def load_config(configdir=None, configfile=None, validate_params=None):
global localconfig
load_defaults(configdir=configdir)
if not configfile:
configfile = os.path.join(localconfig["service_dir"],
DEFAULT_CONFIG_FILENAME)
if not os.path.exists(configfile):
raise Exception("config file (" + str(configfile) + ") not found")
else:
try:
confdata = read_config(configfile=configfile)
update_merge(localconfig, confdata)
except Exception as err:
raise err
try:
validate_config(localconfig, validate_params=validate_params)
except Exception as err:
raise Exception("invalid configuration: details - " + str(err))
# setup service dir
if not os.path.exists(os.path.join(localconfig["service_dir"])):
success = False
for i in range(0, 5):
try:
os.makedirs(os.path.join(localconfig["service_dir"]))
success = True
except:
time.sleep(1)
if not success:
raise Exception("could not create service directory: " +
str(localconfig["service_dir"]))
# setup tmp dir
if not os.path.exists(os.path.join(localconfig["tmp_dir"])):
success = False
for i in range(0, 5):
try:
os.makedirs(os.path.join(localconfig["tmp_dir"]))
success = True
except:
time.sleep(1)
if not success:
raise Exception("could not create temporary directory: " +
str(localconfig["tmp_dir"]))
# copy the src installed files unless they already exist in the service dir conf
load_policy_bundle_paths()
load_filepath_to_config("anchore_scanner_analyzer_config_file",
"analyzer_config.yaml")
# generate/setup the host_id in the service_dir
localconfig["host_id"] = get_host_id()
# any special deployment/environment specific config handling here, via extension config
localconfig["image_content_types"] = image_content_types
localconfig["image_metadata_types"] = image_metadata_types
ext_config = {}
for mod in "anchore_engine", "anchore_enterprise":
try:
ext_config_file = os.path.join(resource_filename(mod, "conf/"),
"extensions.yaml")
except Exception as err:
logger.debug(
"skipping config extension load for module {} - exception: {}".
format(mod, err))
ext_config_file = None
if ext_config_file and os.path.exists(ext_config_file):
try:
with open(ext_config_file, "r") as FH:
d = yaml.safe_load(FH)
if d:
ext_config.update(d)
except Exception as err:
logger.error(
"failed to load extensions.yaml - exception: {}".format(
err))
if ext_config:
if ext_config.get("content_types", []):
localconfig["image_content_types"].extend(
ext_config.get("content_types"))
if ext_config.get("metadata_types", []):
localconfig["image_metadata_types"].extend(
ext_config.get("metadata_types"))
analyzer_config = localconfig.get("services", {}).get("analyzer", {})
if analyzer_config:
localconfig["services"]["analyzer"]["analyzer_driver"] = "nodocker"
return localconfig
def exec(docker_archive, anchore_archive, digest, parent_digest, image_id, tag,
account_id, manifest, dockerfile, created_at, annotation):
"""
Analyze a local image stored as a docker archive (output result of 'docker save'), and generate an anchore image archive tarball ready for import into an anchore engine.
DOCKER_ARCHIVE : Location of input docker archive tarfile to analyze
ANCHORE_ARCHIVE : Location of output anchore image archive to write
"""
global config
# this could be improved to allow use to input timestamps (created_at, analyzed_at, etc)
now = int(time.time())
try:
try:
imageDigest = None
manifest_data = None
rawmanifest = None
if (not manifest and not digest) or (manifest and digest):
raise Exception(
"must supply either an image digest or a valid manifest, but not both"
)
if os.path.exists(anchore_archive):
raise Exception(
"the supplied anchore archive file ({}) already exists, please remove and try again"
.format(anchore_archive))
if manifest:
try:
with open(manifest, 'r') as FH:
# TODO implement manifest validator for anchore requirements, specifically
rawmanifest = FH.read()
input_manifest_data = json.loads(rawmanifest)
imageDigest = manifest_to_digest(rawmanifest)
except Exception as err:
raise ValueError(
"cannot calculate digest from supplied manifest - exception: {}"
.format(err))
if not re.match("^sha256:[\d|a-f]{64}$", imageDigest):
raise ValueError(
"input digest does not validate - must be sha256:<64 hex characters>"
)
if parent_digest:
if re.match("^sha256:[\d|a-f]{64}$", parent_digest):
parentDigest = parent_digest
else:
raise ValueError("input parent_digest does not validate")
else:
parentDigest = imageDigest
if image_id:
if re.match("^[\d|a-f]{64}$", image_id):
imageId = image_id
else:
raise ValueError("input image_id does not validate")
else:
# TODO this could be improved to generate imageId from configuration hash
imageId = "{}".format(''.join(
[random.choice('0123456789abcdef') for x in range(0, 64)]))
if account_id:
userId = account_id
else:
userId = 'admin'
if created_at:
try:
if int(created_at) < 0 or int(created_at) > now + 1:
raise Exception()
except Exception as err:
raise ValueError(
"created_at must by a unix timestamp between 0 and now ({})"
.format(now))
else:
created_at = now
try:
inputTag = tag
image_info = parse_dockerimage_string(inputTag)
fulltag = "{}/{}:{}".format(image_info['registry'],
image_info['repo'],
image_info['tag'])
fulldigest = "{}/{}@{}".format(image_info['registry'],
image_info['repo'], imageDigest)
except Exception as err:
raise ValueError(
"input tag does not validate - exception: {}".format(err))
dockerfile_mode = "Guessed"
dockerfile_contents = None
if dockerfile:
with open(dockerfile, 'r') as FH:
dockerfile_contents = ensure_str(
base64.b64encode(ensure_bytes(FH.read())))
dockerfile_mode = "Actual"
annotations = {}
if annotation:
for a in annotation:
try:
(k, v) = a.split('=', 1)
if k and v:
annotations[k] = v
else:
raise Exception("found null in key or value")
except Exception as err:
raise ValueError(
"annotation format error - annotations must be of the form (--annotation key=value), found: {}"
.format(a))
workspace_root = config['tmp_dir']
except Exception as err:
# input setup/validation failure
raise err
logger.debug(
"input has been prepared: imageDigest={} parentDigest={} imageId={} inputTag={} fulltag={} fulldigest={} userId={} annotations={} created_at={}"
.format(imageDigest, parentDigest, imageId, inputTag, fulltag,
fulldigest, userId, annotations, created_at))
# create an image record
try:
image_record = make_image_record(userId,
'docker',
None,
image_metadata={
'tag': fulltag,
'digest': fulldigest,
'imageId': imageId,
'parentdigest': parentDigest,
'created_at': created_at,
'dockerfile':
dockerfile_contents,
'dockerfile_mode':
dockerfile_mode,
'annotations': annotations
},
registry_lookup=False,
registry_creds=(None, None))
image_record['created_at'] = created_at
image_record['last_updated'] = created_at
image_record['analyzed_at'] = now
image_record['analysis_status'] = 'analyzed'
image_record['image_status'] = 'active'
image_record['record_state_key'] = 'active'
for image_detail in image_record['image_detail']:
image_detail['created_at'] = created_at
image_detail['last_updated'] = created_at
image_detail['tag_detected_at'] = created_at
image_detail['record_state_key'] = 'active'
except Exception as err:
# image record setup fail
raise err
# perform analysis
try:
image_data, analyzed_manifest_data = analyze_image(
userId,
rawmanifest,
image_record,
workspace_root,
config,
registry_creds=[],
use_cache_dir=None,
image_source='docker-archive',
image_source_meta=docker_archive)
image_content_data = {}
for content_type in anchore_engine.common.image_content_types + anchore_engine.common.image_metadata_types:
try:
image_content_data[
content_type] = anchore_engine.common.helpers.extract_analyzer_content(
image_data,
content_type,
manifest=input_manifest_data)
except:
image_content_data[content_type] = {}
anchore_engine.common.helpers.update_image_record_with_analysis_data(
image_record, image_data)
image_record['image_size'] = int(image_record['image_size'])
except Exception as err:
# image analysis fail
raise err
# generate an output image archive tarball
archive_file = anchore_archive
try:
with ImageArchive.for_writing(archive_file) as img_archive:
img_archive.account = userId
img_archive.image_digest = imageDigest
img_archive.manifest.metadata = {
'versions': localconfig.get_versions(),
'image_id': imageId,
'image_record': json.dumps(image_record, sort_keys=True)
}
pack_data = {'document': image_data}
data = ensure_bytes(json.dumps(pack_data, sort_keys=True))
img_archive.add_artifact('analysis',
source=ObjectStoreLocation(
bucket='analysis_data',
key=imageDigest),
data=data,
metadata=None)
pack_data = {'document': image_content_data}
data = ensure_bytes(json.dumps(pack_data, sort_keys=True))
img_archive.add_artifact('image_content',
source=ObjectStoreLocation(
bucket='image_content_data',
key=imageDigest),
data=data,
metadata=None)
pack_data = {'document': input_manifest_data}
data = ensure_bytes(json.dumps(pack_data, sort_keys=True))
img_archive.add_artifact('image_manifest',
source=ObjectStoreLocation(
bucket='manifest_data',
key=imageDigest),
data=data,
metadata=None)
except Exception as err:
# archive tarball generate fail
raise err
except Exception as err:
logger.error(
anchore_manager.cli.utils.format_error_output(
click_config, 'db', {}, err))
sys.exit(2)
click.echo(
"Analysis complete for image {} - archive file is located at {}".
format(imageDigest, archive_file))
def validate_config(config, validate_params=None):
"""
Validate the configuration with required keys and values
:param config: the config dict to validate
:param validate_params: dict of top level config properties and boolean flag
:return: true if passes validation, false otherwise
"""
ret = True
if validate_params is None:
validate_params = default_required_config_params
try:
# ensure there aren't any left over unset variables
confbuf = json.dumps(config)
patt = re.match(".*(\${ANCHORE.*?}).*", confbuf, re.DOTALL)
if patt:
raise Exception(
"variable overrides found in configuration file that are unset ("
+ str(patt.group(1)) + ")")
# top level checks
if 'services' in validate_params and validate_params['services']:
if 'services' not in config or not config['services']:
raise Exception(
"no 'services' definition in configuration file")
else:
for k in list(config['services'].keys()):
if not config['services'][k] or 'enabled' not in config[
'services'][k]:
raise Exception(
"service (" + str(k) +
") defined, but no values are specified (need at least 'enabled: <True|False>')"
)
else:
service_config = config['services'][k]
# check to ensure the listen/port/endpoint_hostname params are set for all services
check_keys = ['endpoint_hostname', 'listen', 'port']
for check_key in check_keys:
if check_key not in service_config:
raise Exception(
"the following values '{}' must be set for all services, but service '{}' does not have them set (missing '{}')"
.format(check_keys, k, check_key))
# check to ensure that if any TLS params are set, then they all must be set
found_key = 0
check_keys = ['ssl_enable', 'ssl_cert', 'ssl_key']
for check_key in check_keys:
if check_key in service_config:
found_key = found_key + 1
if found_key != 0 and found_key != 3:
raise Exception(
"if any one of (" + ','.join(check_keys) +
") are specified, then all must be specified for service '"
+ str(k) + "'")
if 'credentials' in validate_params and validate_params['credentials']:
if 'credentials' not in config or not config['credentials']:
raise Exception(
"no 'credentials' definition in configuration file")
else:
credentials = config['credentials']
for check_key in ['database']:
if check_key not in credentials:
raise Exception(
"no '" + str(check_key) +
"' definition in 'credentials' section of configuration file"
)
elif not credentials[check_key]:
raise Exception(
"'" + str(check_key) +
"' is in configuration file, but is empty (has no records)"
)
# database checks
for check_key in ['db_connect', 'db_connect_args']:
if check_key not in credentials['database']:
raise Exception(
"no '" + str(check_key) +
"' definition in 'credentials'/'database' section of configuration file"
)
# webhook checks
if 'webhooks' in validate_params and validate_params['webhooks']:
if 'webhooks' not in config or not config['webhooks']:
logger.warn(
"no webhooks defined in configuration file - notifications will be disabled"
)
if 'user_authentication' in validate_params and validate_params[
'user_authentication']:
validate_user_auth_config(config)
validate_key_config(config, required=False)
except Exception as err:
logger.error(str(err))
raise err
# raise Exception("TEST")
return (ret)
def get_client(feeds_url=None,
user=None,
conn_timeout=None,
read_timeout=None,
ssl_verify=None):
"""
Returns a configured client based on the local config. Reads configuration from the loaded system configuration.
Uses the admin user's credentials for the feed service if they are available in the external_service_auths/anchoreio/anchorecli/auth json path of the config file. If no specific user credentials are found then the anonymous user credentials are used.
:return: initialize AnchoreIOFeedClient
"""
logger.debug(
"Initializing a feeds client: url={}, user={}, conn_timeout={}, read_timeout={}"
.format(
feeds_url,
user if user is None or type(user) not in [tuple, list]
or len(user) == 0 else (user[0], "***redacted**"),
conn_timeout,
read_timeout,
))
if not (feeds_url and user and conn_timeout and read_timeout):
conf = localconfig.get_config()
if not conf:
logger.error(
"No configuration available. Cannot initialize feed client")
raise ValueError("None for local config")
else:
conf = {
"feeds": {
"connection_timeout_seconds": conn_timeout,
"read_timeout_seconds": read_timeout,
"url": feeds_url,
"ssl_verify": ssl_verify,
}
}
if not conn_timeout:
conn_timeout = conf.get("feeds", {}).get("connection_timeout_seconds")
if not read_timeout:
read_timeout = conf.get("feeds", {}).get("read_timeout_seconds")
if not feeds_url:
feeds_url = conf.get("feeds", {}).get("url")
if not feeds_url:
raise ValueError("no feed service url available")
verify = conf.get("feeds", {}).get("ssl_verify", True)
password = None
if not user:
try:
admin_usr = (conf.get("credentials", {}).get("users", {}).get(
"admin", {}).get("external_service_auths",
{}).get("anchoreio",
{}).get("anchorecli", {}).get("auth"))
if admin_usr:
user, password = admin_usr.split(":")
except AttributeError:
# Something isn't found or was set to None.
pass
else:
user, password = user[0], user[1]
if not user:
user = conf.get("feeds", {}).get("anonymous_user_username")
password = conf.get("feeds", {}).get("anonymous_user_password")
logger.debug("using values: " +
str([feeds_url, user, conn_timeout, read_timeout]))
http_client = HTTPBasicAuthClient(
username=user,
password=password,
connect_timeout=conn_timeout,
read_timeout=read_timeout,
verify=verify,
)
return FeedServiceClient(endpoint=feeds_url, http_client=http_client)
def get_image_metadata_v1(staging_dirs,
imageDigest,
imageId,
manifest_data,
dockerfile_contents="",
dockerfile_mode=""):
outputdir = staging_dirs['outputdir']
unpackdir = staging_dirs['unpackdir']
copydir = staging_dirs['copydir']
docker_history = []
layers = []
dockerfile_mode = "Guessed"
dockerfile_contents = dockerfile_contents
imageArch = ""
try:
imageArch = manifest_data['architecture']
except:
imageArch = ""
try:
for fslayer in manifest_data['fsLayers']:
layers.append(fslayer['blobSum'])
except Exception as err:
logger.error("cannot get layers - exception: " + str(err))
raise err
try:
hfinal = []
count = 0
for rawhel in manifest_data['history']:
hel = json.loads(rawhel['v1Compatibility'])
try:
lsize = hel['Size']
except:
lsize = 0
if hel['container_config']['Cmd']:
lcreatedby = ' '.join(hel['container_config']['Cmd'])
else:
lcreatedby = ""
lcreated = hel['created']
lid = layers[count]
count = count + 1
hfinal.append({
'Created': lcreated,
'CreatedBy': lcreatedby,
'Comment': '',
'Id': lid,
'Size': lsize,
'Tags': []
})
docker_history = hfinal
if hfinal:
with open(os.path.join(unpackdir, "docker_history.json"),
'w') as OFH:
OFH.write(json.dumps(hfinal))
except Exception as err:
logger.error("cannot construct history - exception: " + str(err))
raise err
if not dockerfile_contents:
# get dockerfile_contents (translate history to guessed DF)
# TODO 'FROM' guess?
dockerfile_contents = "FROM scratch\n"
for hel in docker_history:
patt = re.match("^/bin/sh -c #\(nop\) +(.*)", hel['CreatedBy'])
if patt:
cmd = patt.group(1)
elif hel['CreatedBy']:
cmd = "RUN " + hel['CreatedBy']
else:
cmd = None
if cmd:
dockerfile_contents = dockerfile_contents + cmd + "\n"
dockerfile_mode = "Guessed"
elif not dockerfile_mode:
dockerfile_mode = "Actual"
layers.reverse()
return (docker_history, layers, dockerfile_contents, dockerfile_mode,
imageArch)
def download_image(fulltag,
copydir,
user=None,
pw=None,
verify=True,
manifest=None,
use_cache_dir=None,
dest_type='oci'):
try:
proc_env = os.environ.copy()
if user and pw:
proc_env['SKOPUSER'] = user
proc_env['SKOPPASS'] = pw
credstr = '--src-creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
else:
credstr = ""
if verify:
tlsverifystr = "--src-tls-verify=true"
else:
tlsverifystr = "--src-tls-verify=false"
if use_cache_dir and os.path.exists(use_cache_dir):
cachestr = "--dest-shared-blob-dir " + use_cache_dir
else:
cachestr = ""
if dest_type == 'oci':
if manifest:
with open(os.path.join(copydir, "manifest.json"), 'w') as OFH:
OFH.write(manifest)
cmd = [
"/bin/sh", "-c",
"skopeo copy {} {} {} docker://{} oci:{}:image".format(
tlsverifystr, credstr, cachestr, fulltag, copydir)
]
else:
cmd = [
"/bin/sh", "-c", "skopeo copy {} {} docker://{} dir:{}".format(
tlsverifystr, credstr, fulltag, copydir)
]
cmdstr = ' '.join(cmd)
try:
rc, sout, serr = anchore_engine.services.common.run_command_list(
cmd, env=proc_env)
if rc != 0:
raise Exception("command failed: cmd=" + str(cmdstr) +
" exitcode=" + str(rc) + " stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
else:
logger.debug("command succeeded: cmd=" + str(cmdstr) +
" stdout=" + str(sout).strip() + " stderr=" +
str(serr).strip())
except Exception as err:
logger.error("command failed with exception - " + str(err))
raise err
except Exception as err:
raise err
return (True)
def get_image_manifest(userId, image_info, registry_creds):
logger.debug("get_image_manifest input: " + str(userId) + " : " +
str(image_info) + " : " + str(time.time()))
user = pw = None
repo = url = None
registry_verify = True
registry = image_info['registry']
try:
user, pw, registry_verify = anchore_engine.auth.common.get_creds_by_registry(
registry, registry_creds=registry_creds)
except Exception as err:
raise err
if registry == 'docker.io':
url = "https://index.docker.io"
if not re.match(".*/.*", image_info['repo']):
repo = "library/" + image_info['repo']
else:
repo = image_info['repo']
else:
url = "https://" + registry
repo = image_info['repo']
if image_info['digest']:
tag = None
input_digest = image_info['digest']
fulltag = "{}/{}@{}".format(registry, repo, input_digest)
else:
input_digest = None
tag = image_info['tag']
fulltag = "{}/{}:{}".format(registry, repo, tag)
manifest = digest = None
logger.debug("trying to get manifest/digest for image (" + str(fulltag) +
")")
err = None
try:
if tag:
manifest, digest = get_image_manifest_skopeo(
url,
registry,
repo,
intag=tag,
user=user,
pw=pw,
verify=registry_verify)
elif input_digest:
manifest, digest = get_image_manifest_skopeo(
url,
registry,
repo,
indigest=input_digest,
user=user,
pw=pw,
verify=registry_verify)
else:
raise Exception("neither tag nor digest was given as input")
except Exception as err:
logger.error("could not fetch manifest/digest: " + str(err))
manifest = digest = None
if manifest and digest:
return (manifest, digest)
logger.error(
"could not get manifest/digest for image ({}) from registry ({}) - error: {}"
.format(fulltag, url, err))
raise Exception(
"could not get manifest/digest for image ({}) from registry ({}) - error: {}"
.format(fulltag, url, err))
return ({}, "")
def lookup_registry_image(self, tag=None, digest=None):
if not tag and not digest:
logger.error("no input (tag=, digest=)")
raise Exception("bad input")
return self.call_api(http.anchy_get, 'registry_lookup', query_params={'digest': digest, 'tag': tag})
def get_image_manifest_orig(userId, image_info, registry_creds):
logger.debug("get_image_manifest input: " + str(userId) + " : " +
str(image_info) + " : " + str(time.time()))
user = pw = None
registry_verify = True
registry = image_info['registry']
try:
user, pw, registry_verify = anchore_engine.auth.common.get_creds_by_registry(
registry, registry_creds=registry_creds)
except Exception as err:
raise err
if False:
# replaced with utility func above
try:
for registry_record in registry_creds:
if registry_record['registry'] == registry:
if registry_record['record_state_key'] not in ['active']:
try:
last_try = int(registry_record['record_state_val'])
except:
last_try = 0
if (int(time.time()) - last_try) < 60:
logger.debug(
"SKIPPING REGISTRY ATTEMPT: " +
str(registry_record['record_state_key']))
raise Exception(
"registry not available - " +
str(registry_record['record_state_key']))
user, pw = anchore_engine.auth.common.get_docker_registry_userpw(
registry_record)
registry_verify = registry_record['registry_verify']
break
except Exception as err:
raise err
if registry == 'docker.io':
url = "https://index.docker.io"
if not re.match(".*/.*", image_info['repo']):
repo = "library/" + image_info['repo']
else:
repo = image_info['repo']
else:
url = "https://" + registry
repo = image_info['repo']
oauth_err = basicauth_err = skopeoauth_err = "N/A"
if image_info['digest']:
pullstring = image_info['digest']
else:
pullstring = image_info['tag']
auth_funcs = [
get_image_manifest_oauth2, get_image_manifest_docker_registry,
get_image_manifest_skopeo
]
auth_errors = {}
manifest = digest = None
for af in auth_funcs:
manifest = digest = None
try:
imagestr = url + "/" + repo + ":" + pullstring
except:
imagestr = pullstring
logger.debug("trying to get manifest/digest for image (" +
str(imagestr) + ") using (" + str(af.__name__) + ")")
try:
manifest, digest = af(url,
registry,
repo,
pullstring,
user=user,
pw=pw,
verify=registry_verify)
except Exception as err:
logger.debug("could not get manifest/digest for image (" +
imagestr + ") using (" + str(af.__name__) +
") - exception: " + str(err))
auth_errors[af.__name__] = str(err)
if manifest and digest:
break
if manifest and digest:
return (manifest, digest)
logger.error(
"could not get manifest/digest for image using any auth method: (" +
str(pullstring) + "): " + str(auth_errors))
raise Exception(
"could not get manifest/digest for image using any auth method: (" +
str(pullstring) + "): " + str(auth_errors))
return ({}, "")
def handle_image_analyzer(*args, **kwargs):
global system_user_auth, queuename, servicename
cycle_timer = kwargs['mythread']['cycle_timer']
localconfig = anchore_engine.configuration.localconfig.get_config()
system_user_auth = localconfig['system_user_auth']
threads = []
layer_cache_dirty = True
while (True):
logger.debug("analyzer thread cycle start")
try:
myconfig = localconfig['services']['analyzer']
max_analyze_threads = int(myconfig.get('max_threads', 1))
layer_cache_enable = myconfig.get('layer_cache_enable', False)
logger.debug("max threads: " + str(max_analyze_threads))
if len(threads) < max_analyze_threads:
logger.debug("analyzer has free worker threads {} / {}".format(
len(threads), max_analyze_threads))
qobj = simplequeue.dequeue(system_user_auth, queuename)
if qobj:
logger.debug("got work from queue task Id: {}".format(
qobj.get('queueId', 'unknown')))
myqobj = copy.deepcopy(qobj)
logger.spew("incoming queue object: " + str(myqobj))
logger.debug("incoming queue task: " + str(myqobj.keys()))
logger.debug("starting thread")
athread = threading.Thread(target=process_analyzer_job,
args=(system_user_auth, myqobj,
layer_cache_enable))
athread.start()
threads.append(athread)
logger.debug("thread started")
layer_cache_dirty = True
else:
logger.debug(
"analyzer queue is empty - no work this cycle")
else:
logger.debug("all workers are busy")
alive_threads = []
while (threads):
athread = threads.pop()
if not athread.isAlive():
try:
logger.debug("thread completed - joining")
athread.join()
logger.debug("thread joined")
except Exception as err:
logger.warn("cannot join thread - exception: " +
str(err))
else:
alive_threads.append(athread)
threads = alive_threads
if layer_cache_enable and layer_cache_dirty and len(threads) == 0:
logger.debug("running layer cache handler")
try:
handle_layer_cache()
layer_cache_dirty = False
except Exception as err:
logger.warn("layer cache management failed - exception: " +
str(err))
except Exception as err:
import traceback
traceback.print_exc()
logger.error(str(err))
logger.debug("analyzer thread cycle complete: next in " +
str(cycle_timer))
time.sleep(cycle_timer)
return (True)
def notify(self, notification_type, notification_value):
"""
No-Op for the default handler since permissions are ephemeral.
:param notification_type:
:param notification_value:
:return:
"""
logger.info('Calling notification!')
retries = 3
try:
if not ExternalAuthzRealm.__client__:
logger.warn(
'Got authz notification type: {} value:{}, but no client configured so nothing to do'
.format(notification_type, notification_value))
return True
else:
if NotificationTypes.domain_created == notification_type:
fn = ExternalAuthzRealm.__client__.initialize_domain
elif NotificationTypes.domain_deleted == notification_type:
fn = ExternalAuthzRealm.__client__.delete_domain
elif NotificationTypes.principal_created == notification_type:
fn = ExternalAuthzRealm.__client__.initialize_principal
elif NotificationTypes.principal_deleted == notification_type:
fn = ExternalAuthzRealm.__client__.delete_principal
else:
fn = None
if fn is None:
logger.warn(
'Got notification type {} with no handler mapped'.format(
notification_type))
return
err = None
for i in range(retries):
try:
resp = fn(notification_value)
if not resp:
logger.warn(
'Bad response from authz service, will retry: {}'.
format(resp))
else:
logger.debug(
'Notification succeeded to authz plugin service')
break
except Exception as ex:
err = ex
logger.exception(
'Error calling {} against authz plugin client'.format(
fn.__name__))
else:
logger.error(
'Could not confirm successful response of authz handler for notification {} with value {}'
.format(notification_type, notification_value))
raise Exception(
'Error invoking POST /domains on external authz handler: {}'
.format(
str(err) if err else 'Retry count exceeded {}'.
format(retries)))
return True
except:
logger.exception(
'Notification handler for external authz plugin caught exception and could not complete: {} {}'
.format(notification_type, notification_type))
raise
def process_analyzer_job(system_user_auth, qobj, layer_cache_enable):
global servicename #current_avg, current_avg_count
timer = int(time.time())
event = None
try:
logger.debug('dequeued object: {}'.format(qobj))
record = qobj['data']
userId = record['userId']
imageDigest = record['imageDigest']
manifest = record['manifest']
user_record = catalog.get_user(system_user_auth, userId)
user_auth = (user_record['userId'], user_record['password'])
# check to make sure image is still in DB
try:
image_records = catalog.get_image(user_auth,
imageDigest=imageDigest)
if image_records:
image_record = image_records[0]
else:
raise Exception("empty image record from catalog")
except Exception as err:
logger.warn(
"dequeued image cannot be fetched from catalog - skipping analysis ("
+ str(imageDigest) + ") - exception: " + str(err))
return (True)
logger.info("image dequeued for analysis: " + str(userId) + " : " +
str(imageDigest))
if image_record[
'analysis_status'] != anchore_engine.subsys.taskstate.base_state(
'analyze'):
logger.debug(
"dequeued image is not in base state - skipping analysis")
return (True)
try:
logger.spew("TIMING MARK0: " + str(int(time.time()) - timer))
last_analysis_status = image_record['analysis_status']
image_record[
'analysis_status'] = anchore_engine.subsys.taskstate.working_state(
'analyze')
rc = catalog.update_image(user_auth, imageDigest, image_record)
# disable the webhook call for image state transistion to 'analyzing'
#try:
# for image_detail in image_record['image_detail']:
# fulltag = image_detail['registry'] + "/" + image_detail['repo'] + ":" + image_detail['tag']
# npayload = {
# 'last_eval': {'imageDigest': imageDigest, 'analysis_status': last_analysis_status},
# 'curr_eval': {'imageDigest': imageDigest, 'analysis_status': image_record['analysis_status']},
# }
# rc = anchore_engine.subsys.notifications.queue_notification(userId, fulltag, 'analysis_update', npayload)
#except Exception as err:
# logger.warn("failed to enqueue notification on image analysis state update - exception: " + str(err))
# actually do analysis
registry_creds = catalog.get_registry(user_auth)
try:
image_data = perform_analyze(
userId,
manifest,
image_record,
registry_creds,
layer_cache_enable=layer_cache_enable)
except AnchoreException as e:
event = events.AnalyzeImageFail(user_id=userId,
image_digest=imageDigest,
error=e.to_dict())
raise
imageId = None
try:
imageId = image_data[0]['image']['imageId']
except Exception as err:
logger.warn(
"could not get imageId after analysis or from image record - exception: "
+ str(err))
try:
logger.debug("archiving analysis data")
rc = catalog.put_document(user_auth, 'analysis_data',
imageDigest, image_data)
except Exception as e:
err = CatalogClientError(
msg='Failed to upload analysis data to catalog', cause=e)
event = events.ArchiveAnalysisFail(user_id=userId,
image_digest=imageDigest,
error=err.to_dict())
raise err
if rc:
try:
logger.debug("extracting image content data")
image_content_data = {}
for content_type in anchore_engine.services.common.image_content_types + anchore_engine.services.common.image_metadata_types:
try:
image_content_data[
content_type] = anchore_engine.services.common.extract_analyzer_content(
image_data,
content_type,
manifest=manifest)
except:
image_content_data[content_type] = {}
if image_content_data:
logger.debug("adding image content data to archive")
rc = catalog.put_document(user_auth,
'image_content_data',
imageDigest,
image_content_data)
try:
logger.debug(
"adding image analysis data to image_record")
anchore_engine.services.common.update_image_record_with_analysis_data(
image_record, image_data)
except Exception as err:
raise err
except Exception as err:
logger.warn(
"could not store image content metadata to archive - exception: "
+ str(err))
logger.debug("adding image record to policy-engine service (" +
str(userId) + " : " + str(imageId) + ")")
try:
if not imageId:
raise Exception(
"cannot add image to policy engine without an imageId"
)
localconfig = anchore_engine.configuration.localconfig.get_config(
)
verify = localconfig['internal_ssl_verify']
client = anchore_engine.clients.policy_engine.get_client(
user=system_user_auth[0],
password=system_user_auth[1],
verify_ssl=verify)
try:
logger.debug(
"clearing any existing record in policy engine for image: "
+ str(imageId))
rc = client.delete_image(user_id=userId,
image_id=imageId)
except Exception as err:
logger.warn("exception on pre-delete - exception: " +
str(err))
logger.info('Loading image: {} {}'.format(userId, imageId))
request = ImageIngressRequest(
user_id=userId,
image_id=imageId,
fetch_url='catalog://' + str(userId) +
'/analysis_data/' + str(imageDigest))
logger.debug("policy engine request: " + str(request))
resp = client.ingress_image(request)
logger.debug("policy engine image add response: " +
str(resp))
except Exception as err:
import traceback
traceback.print_exc()
newerr = PolicyEngineClientError(
msg='Adding image to policy-engine failed',
cause=str(err))
event = events.LoadAnalysisFail(user_id=userId,
image_digest=imageDigest,
error=newerr.to_dict())
raise newerr
logger.debug("updating image catalog record analysis_status")
last_analysis_status = image_record['analysis_status']
image_record[
'analysis_status'] = anchore_engine.subsys.taskstate.complete_state(
'analyze')
image_record['analyzed_at'] = int(time.time())
rc = catalog.update_image(user_auth, imageDigest, image_record)
try:
annotations = {}
try:
if image_record.get('annotations', '{}'):
annotations = json.loads(
image_record.get('annotations', '{}'))
except Exception as err:
logger.warn(
"could not marshal annotations from json - exception: "
+ str(err))
for image_detail in image_record['image_detail']:
fulltag = image_detail['registry'] + "/" + image_detail[
'repo'] + ":" + image_detail['tag']
last_payload = {
'imageDigest': imageDigest,
'analysis_status': last_analysis_status,
'annotations': annotations
}
curr_payload = {
'imageDigest': imageDigest,
'analysis_status': image_record['analysis_status'],
'annotations': annotations
}
npayload = {
'last_eval': last_payload,
'curr_eval': curr_payload,
}
if annotations:
npayload['annotations'] = annotations
rc = anchore_engine.subsys.notifications.queue_notification(
userId, fulltag, 'analysis_update', npayload)
except Exception as err:
logger.warn(
"failed to enqueue notification on image analysis state update - exception: "
+ str(err))
else:
err = CatalogClientError(
msg='Failed to upload analysis data to catalog',
cause='Invalid response from catalog API - {}'.format(
str(rc)))
event = events.ArchiveAnalysisFail(user_id=userId,
image_digest=imageDigest,
error=err.to_dict())
raise err
logger.info("analysis complete: " + str(userId) + " : " +
str(imageDigest))
logger.spew("TIMING MARK1: " + str(int(time.time()) - timer))
try:
run_time = float(time.time() - timer)
#current_avg_count = current_avg_count + 1.0
#new_avg = current_avg + ((run_time - current_avg) / current_avg_count)
#current_avg = new_avg
anchore_engine.subsys.metrics.histogram_observe(
'anchore_analysis_time_seconds',
run_time,
buckets=[
1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0,
1800.0, 3600.0
],
status="success")
#anchore_engine.subsys.metrics.counter_inc('anchore_images_analyzed_total')
#localconfig = anchore_engine.configuration.localconfig.get_config()
#service_record = {'hostid': localconfig['host_id'], 'servicename': servicename}
#anchore_engine.subsys.servicestatus.set_status(service_record, up=True, available=True, detail={'avg_analysis_time_sec': current_avg, 'total_analysis_count': current_avg_count}, update_db=True)
except Exception as err:
logger.warn(str(err))
pass
except Exception as err:
run_time = float(time.time() - timer)
logger.exception("problem analyzing image - exception: " +
str(err))
anchore_engine.subsys.metrics.histogram_observe(
'anchore_analysis_time_seconds',
run_time,
buckets=[
1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0,
3600.0
],
status="fail")
image_record[
'analysis_status'] = anchore_engine.subsys.taskstate.fault_state(
'analyze')
image_record[
'image_status'] = anchore_engine.subsys.taskstate.fault_state(
'image_status')
rc = catalog.update_image(user_auth, imageDigest, image_record)
finally:
if event:
try:
catalog.add_event(user_auth, event)
except:
logger.error(
'Ignoring error creating analysis failure event')
except Exception as err:
logger.warn("job processing bailed - exception: " + str(err))
raise err
return (True)
def perform_analyze_localanchore(userId,
manifest,
image_record,
registry_creds,
layer_cache_enable=False):
ret_analyze = {}
localconfig = anchore_engine.configuration.localconfig.get_config()
do_docker_cleanup = localconfig['cleanup_images']
try:
image_detail = image_record['image_detail'][0]
registry_manifest = manifest
pullstring = image_detail['registry'] + "/" + image_detail[
'repo'] + "@" + image_detail['imageDigest']
fulltag = image_detail['registry'] + "/" + image_detail[
'repo'] + ":" + image_detail['tag']
logger.debug("using pullstring (" + str(pullstring) +
") and fulltag (" + str(fulltag) + ") to pull image data")
except Exception as err:
image_detail = pullstring = fulltag = None
raise Exception(
"failed to extract requisite information from image_record - exception: "
+ str(err))
timer = int(time.time())
logger.spew("TIMING MARK0: " + str(int(time.time()) - timer))
logger.debug("obtaining anchorelock..." + str(pullstring))
with localanchore.get_anchorelock(lockId=pullstring):
logger.debug("obtaining anchorelock successful: " + str(pullstring))
logger.spew("TIMING MARK1: " + str(int(time.time()) - timer))
logger.info("performing analysis on image: " + str(pullstring))
# pull the digest, but also any tags associated with the image (that we know of) in order to populate the local docker image
try:
rc = localanchore.pull(userId,
pullstring,
image_detail,
pulltags=True,
registry_creds=registry_creds)
if not rc:
raise Exception("anchore analyze failed:")
pullstring = re.sub("sha256:", "", rc['Id'])
image_detail['imageId'] = pullstring
except Exception as err:
logger.error("error on pull: " + str(err))
raise err
logger.spew("TIMING MARK2: " + str(int(time.time()) - timer))
# analyze!
try:
rc = localanchore.analyze(pullstring, image_detail)
if not rc:
raise Exception("anchore analyze failed:")
except Exception as err:
logger.error("error on analyze: " + str(err))
raise err
logger.spew("TIMING MARK3: " + str(int(time.time()) - timer))
# get the result from anchore
logger.debug("retrieving image data from anchore")
try:
image_data = localanchore.get_image_export(pullstring,
image_detail)
if not image_data:
raise Exception("anchore image data export failed:")
except Exception as err:
logger.error("error on image export: " + str(err))
raise err
logger.spew("TIMING MARK5: " + str(int(time.time()) - timer))
try:
logger.debug("removing image: " + str(pullstring))
rc = localanchore.remove_image(pullstring,
docker_remove=do_docker_cleanup,
anchore_remove=True)
logger.debug("removing image complete: " + str(pullstring))
except Exception as err:
raise err
logger.spew("TIMING MARK6: " + str(int(time.time()) - timer))
ret_analyze = image_data
logger.info("performing analysis on image complete: " + str(pullstring))
return (ret_analyze)
def get_image_manifest_skopeo(url,
registry,
repo,
intag=None,
indigest=None,
user=None,
pw=None,
verify=True):
manifest = {}
digest = None
testDigest = None
if indigest:
pullstring = registry + "/" + repo + "@" + indigest
elif intag:
pullstring = registry + "/" + repo + ":" + intag
else:
raise Exception(
"invalid input - must supply either an intag or indigest")
try:
if user and pw:
os.environ['SKOPUSER'] = user
os.environ['SKOPPASS'] = pw
credstr = "--creds ${SKOPUSER}:${SKOPPASS}"
credstr = "--creds " + user + ":" + pw
else:
credstr = ""
if verify:
tlsverifystr = "--tls-verify=true"
else:
tlsverifystr = "--tls-verify=false"
try:
cmdstr = "skopeo inspect --raw " + tlsverifystr + " " + credstr + " docker://" + pullstring
try:
rc, sout, serr = anchore_engine.services.common.run_command(
cmdstr)
if rc != 0:
raise Exception("command failed: cmd=" + str(cmdstr) +
" exitcode=" + str(rc) + " stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
else:
logger.debug("command succeeded: cmd=" + str(cmdstr) +
" stdout=" + str(sout).strip() + " stderr=" +
str(serr).strip())
#testDigest = anchore_engine.services.common.manifest_to_digest(sout)
except Exception as err:
logger.error("command failed with exception - " + str(err))
raise err
digest = anchore_engine.services.common.manifest_to_digest(sout)
manifest = json.loads(sout)
except Exception as err:
logger.warn("CMD failed - exception: " + str(err))
digest = None
manifest = {}
#try:
# cmdstr = "skopeo inspect "+tlsverifystr+" "+credstr+" docker://"+pullstring
# try:
# rc, sout, serr = anchore_engine.services.common.run_command(cmdstr)
# if rc != 0:
# raise Exception("command failed: cmd="+str(cmdstr)+" exitcode="+str(rc)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip())
# else:
# logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip())
# except Exception as err:
# logger.error("command failed with exception - " + str(err))
# raise err
# skopeo_output = sout
# data = json.loads(skopeo_output)
# digest = data['Digest']
#except Exception as err:
# logger.warn("CMD failed - exception: " + str(err))
# digest = None
except Exception as err:
#logger.error("error in skopeo wrapper - exception: " + str(err))
raise err
finally:
try:
del os.environ['SKOPUSER']
except:
pass
try:
del os.environ['SKOPPASS']
except:
pass
if not manifest or not digest:
raise Exception("no digest/manifest from skopeo")
#logger.debug("digest test comparison: " + str(digest == testDigest) + " : " + str(digest) + " : " + str(testDigest))
return (manifest, digest)
def default_monitor_func(**kwargs):
"""
Generic monitor thread function for invoking tasks defined in a monitor dict
:param kwargs:
:return:
"""
global click, running, last_run
my_monitors = kwargs['monitors']
monitor_threads = kwargs['monitor_threads']
servicename = kwargs['servicename']
timer = int(time.time())
if click < 5:
click = click + 1
logger.debug("service ("+str(servicename)+") starting in: " + str(5 - click))
return True
if round(time.time() - last_run) < kwargs['kick_timer']:
logger.spew(
"timer hasn't kicked yet: " + str(round(time.time() - last_run)) + " : " + str(kwargs['kick_timer']))
return True
try:
running = True
last_run = time.time()
# handle setting the cycle timers based on configuration
for monitor_name in list(my_monitors.keys()):
if not my_monitors[monitor_name]['initialized']:
# first time
if 'cycle_timers' in kwargs and monitor_name in kwargs['cycle_timers']:
try:
the_cycle_timer = my_monitors[monitor_name]['cycle_timer']
min_cycle_timer = my_monitors[monitor_name]['min_cycle_timer']
max_cycle_timer = my_monitors[monitor_name]['max_cycle_timer']
config_cycle_timer = int(kwargs['cycle_timers'][monitor_name])
if config_cycle_timer < 0:
the_cycle_timer = abs(int(config_cycle_timer))
elif config_cycle_timer == 0:
my_monitors[monitor_name]['enabled'] = False
logger.debug("monitor '{}' has been explicitly disabled in config".format(monitor_name))
elif config_cycle_timer < min_cycle_timer:
logger.warn("configured cycle timer for handler ("+str(monitor_name)+") is less than the allowed min ("+str(min_cycle_timer)+") - using allowed min")
the_cycle_timer = min_cycle_timer
elif config_cycle_timer > max_cycle_timer:
logger.warn("configured cycle timer for handler ("+str(monitor_name)+") is greater than the allowed max ("+str(max_cycle_timer)+") - using allowed max")
the_cycle_timer = max_cycle_timer
else:
the_cycle_timer = config_cycle_timer
my_monitors[monitor_name]['cycle_timer'] = the_cycle_timer
except Exception as err:
logger.warn("exception setting custom cycle timer for handler ("+str(monitor_name)+") - using default")
my_monitors[monitor_name]['initialized'] = True
# handle the thread (re)starters here
for monitor_name in list(my_monitors.keys()):
if my_monitors[monitor_name].get('enabled', True):
start_thread = False
if monitor_name not in monitor_threads:
start_thread = True
else:
if not monitor_threads[monitor_name].isAlive():
logger.debug("thread stopped - restarting: " + str(monitor_name))
monitor_threads[monitor_name].join()
start_thread = True
if start_thread:
monitor_threads[monitor_name] = threading.Thread(target=my_monitors[monitor_name]['handler'], args=my_monitors[monitor_name]['args'], kwargs={'mythread': my_monitors[monitor_name]})
logger.debug("starting up monitor_thread: " + str(monitor_name))
monitor_threads[monitor_name].start()
except Exception as err:
logger.error(str(err))
finally:
running = False
return True
def get_image_manifest_dockerhub_orig(repo, tag, user=None, pw=None):
manifest = {}
digest = ""
try:
if not user or not pw:
authy = None
else:
authy = (user, pw)
#TODO externalize URLs
auth_url = "https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repository}:pull"
url = auth_url.format(repository=repo)
token = ""
try:
r = requests.get(url, json=True, auth=authy)
if r.status_code == 200:
#token = requests.get(url, json=True, auth=authy).json()["token"]
token = r.json()["token"]
elif r.status_code == 402:
raise Exception(
"not authorized (401) returned from registry: auth_url=(" +
str(url) + ") user=(" + str(user) + ")")
else:
raise Exception("got bad code (" + str(r.status_code) +
") from manifest request: " + str(r.text))
except Exception as err:
logger.error("could not get auth token: " + str(err))
raise err
get_manifest_template = "https://registry.hub.docker.com/v2/{repository}/manifests/{tag}"
url = get_manifest_template.format(repository=repo, tag=tag)
try:
headers = {
"Authorization": "Bearer {}".format(token),
"Accept":
"application/vnd.docker.distribution.manifest.v2+json"
}
r = requests.get(url, headers=headers, json=True)
if r.status_code == 200:
manifest = r.json()
digest = r.headers['Docker-Content-Digest']
elif r.status_code == 401:
raise Exception(
"not authorized (401) returned from registry: registry=(https://registry.hub.docker.com) repo=("
+ str(repo) + ") tag=(" + str(tag) + ") user=(" +
str(user) + ")")
else:
raise Exception("got bad code (" + str(r.status_code) +
") from manifest request: " + str(r.text))
except Exception as err:
logger.warn("could not get manifest: " + str(err))
raise err
except Exception as err:
raise err
return (manifest, digest)
def get_image_manifest_skopeo(url,
registry,
repo,
intag=None,
indigest=None,
user=None,
pw=None,
verify=True):
manifest = {}
digest = None
testDigest = None
if indigest:
pullstring = registry + "/" + repo + "@" + indigest
elif intag:
pullstring = registry + "/" + repo + ":" + intag
else:
raise Exception(
"invalid input - must supply either an intag or indigest")
try:
proc_env = os.environ.copy()
if user and pw:
proc_env['SKOPUSER'] = user
proc_env['SKOPPASS'] = pw
credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
else:
credstr = ""
if verify:
tlsverifystr = "--tls-verify=true"
else:
tlsverifystr = "--tls-verify=false"
try:
cmd = [
"/bin/sh", "-c",
"skopeo inspect --raw {} {} docker://{}".format(
tlsverifystr, credstr, pullstring)
]
cmdstr = ' '.join(cmd)
try:
rc, sout, serr = anchore_engine.services.common.run_command_list(
cmd, env=proc_env)
if rc != 0:
raise Exception("command failed: cmd=" + str(cmdstr) +
" exitcode=" + str(rc) + " stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
else:
logger.debug("command succeeded: cmd=" + str(cmdstr) +
" stdout=" + str(sout).strip() + " stderr=" +
str(serr).strip())
except Exception as err:
logger.error("command failed with exception - " + str(err))
raise err
digest = anchore_engine.services.common.manifest_to_digest(sout)
manifest = json.loads(sout)
except Exception as err:
logger.warn("CMD failed - exception: " + str(err))
digest = None
manifest = {}
except Exception as err:
raise err
if not manifest or not digest:
raise Exception("no digest/manifest from skopeo")
return (manifest, digest)
def makeService(snames,
options,
db_connect=True,
require_system_user_auth=True,
module_name="anchore_engine.services",
validate_params={}):
try:
logger.enable_bootstrap_logging(service_name=','.join(snames))
try:
# config and init
configfile = configdir = None
if options['config']:
configdir = options['config']
configfile = os.path.join(options['config'], 'config.yaml')
anchore_engine.configuration.localconfig.load_config(
configdir=configdir,
configfile=configfile,
validate_params=validate_params)
localconfig = anchore_engine.configuration.localconfig.get_config()
localconfig['myservices'] = []
logger.spew("localconfig=" +
json.dumps(localconfig, indent=4, sort_keys=True))
except Exception as err:
logger.error("cannot load configuration: exception - " + str(err))
raise err
# get versions of things
try:
versions = anchore_engine.configuration.localconfig.get_versions()
except Exception as err:
logger.error("cannot detect versions of service: exception - " +
str(err))
raise err
if db_connect:
logger.info("initializing database")
# connect to DB
try:
db.initialize(localconfig=localconfig, versions=versions)
except Exception as err:
logger.error("cannot connect to configured DB: exception - " +
str(err))
raise err
#credential bootstrap
localconfig['system_user_auth'] = (None, None)
if require_system_user_auth:
gotauth = False
max_retries = 60
for count in range(1, max_retries):
if gotauth:
continue
try:
with session_scope() as dbsession:
localconfig[
'system_user_auth'] = get_system_user_auth(
session=dbsession)
if localconfig['system_user_auth'] != (None, None):
gotauth = True
else:
logger.error(
"cannot get system user auth credentials yet, retrying ("
+ str(count) + " / " + str(max_retries) + ")")
time.sleep(5)
except Exception as err:
logger.error(
"cannot get system-user auth credentials - service may not have system level access"
)
localconfig['system_user_auth'] = (None, None)
if not gotauth:
raise Exception(
"service requires system user auth to start")
# application object
application = service.Application("multi-service-" + '-'.join(snames))
#multi-service
retservice = service.MultiService()
retservice.setServiceParent(application)
success = False
try:
scount = 0
for sname in snames:
if sname in localconfig['services'] and localconfig[
'services'][sname]['enabled']:
smodule = importlib.import_module(module_name + "." +
sname)
s = smodule.createService(sname, localconfig)
s.setServiceParent(retservice)
rc = smodule.initializeService(sname, localconfig)
if not rc:
raise Exception("failed to initialize service")
rc = smodule.registerService(sname, localconfig)
if not rc:
raise Exception("failed to register service")
logger.debug("starting service: " + sname)
success = True
scount += 1
localconfig['myservices'].append(sname)
else:
logger.error(
"service not enabled in config, not starting service: "
+ sname)
if scount == 0:
logger.error(
"no services/subservices were enabled/started on this host"
)
success = False
except Exception as err:
logger.error("cannot create/init/register service: " + sname +
" - exception: " + str(err))
success = False
if not success:
logger.error("cannot start service (see above for information)")
traceback.print_exc('Service init failure')
raise Exception("cannot start service (see above for information)")
return (retservice)
finally:
logger.disable_bootstrap_logging()
def get_authenticated_cli(userId, registry, registry_creds=[]):
global docker_cli_unauth, docker_clis
logger.debug(
"DOCKER CLI: entering auth cli create/fetch for input user/registry: "
+ str(userId) + " / " + str(registry))
localconfig = anchore_engine.configuration.localconfig.get_config()
if not userId:
if not docker_cli_unauth:
docker_cli_unauth = docker.Client(
base_url=localconfig['docker_conn'],
version='auto',
timeout=int(localconfig['docker_conn_timeout']))
logger.debug("DOCKER CLI: returning unauth client")
return (docker_cli_unauth)
if userId in docker_clis and registry in docker_clis[userId]:
if 'registry_creds' in docker_clis[userId][
registry] and registry_creds == docker_clis[userId][registry][
'registry_creds']:
logger.debug("DOCKER CLI: found existing authenticated CLI")
return (docker_clis[userId][registry]['cli'])
else:
logger.debug("DOCKER CLI: detected cred change, will refresh CLI")
logger.debug("DOCKER CLI: making new auth CLI for user/registry: " +
str(userId) + " / " + str(registry))
try:
if userId not in docker_clis:
docker_clis[userId] = {}
if registry not in docker_clis[userId]:
docker_clis[userId][registry] = {}
user = pw = None
for registry_record in registry_creds:
if registry_record['registry'] == registry:
user, pw = anchore_engine.auth.common.get_docker_registry_userpw(
registry_record)
if not user or not pw:
logger.debug("DOCKER CLI: making unauth CLI")
docker_clis[userId][registry]['cli'] = docker.Client(
base_url=localconfig['docker_conn'],
version='auto',
timeout=int(localconfig['docker_conn_timeout']))
docker_clis[userId][registry]['registry_creds'] = []
else:
logger.debug("DOCKER CLI: making auth CLI")
try:
cli = docker.Client(base_url=localconfig['docker_conn'],
version='auto',
timeout=int(
localconfig['docker_conn_timeout']))
rc = cli.login(user,
password=pw,
registry=registry,
reauth=False)
docker_clis[userId][registry]['cli'] = cli
docker_clis[userId][registry][
'registry_creds'] = registry_creds
except Exception as err:
logger.error("DOCKER CLI auth err: " + str(err))
raise err
except Exception as err:
logger.error("DOCKER CLI: unable to get docker cli - exception: " +
str(err))
raise err
if userId in docker_clis and registry in docker_clis[userId]:
logger.debug("DOCKER CLI: returning auth client")
return (docker_clis[userId][registry]['cli'])
logger.error(
"DOCKER CLI: unable to complete authenticated client create/fetch")
raise Exception(
"DOCKER CLI: unable to complete authenticated client create/fetch")
return (None)
def sync_metadata(
source_feeds: Dict[
str, Dict[str, Union[FeedAPIRecord, List[FeedAPIGroupRecord]]]
],
to_sync: List[str] = None,
operation_id: Optional[str] = None,
groups: bool = True,
) -> Tuple[Dict[str, FeedMetadata], List[Tuple[str, Union[str, BaseException]]]]:
"""
Get metadata from source and sync db metadata records to that (e.g. add any new groups or feeds)
Executes as a unit-of-work for db, so will commit result and returns the records found on upstream source.
If a record exists in db but was not found upstream, it is not returned
:param source_feeds: mapping containing FeedAPIRecord and FeedAPIGroupRecord
:type source_feeds: Dict[str, Dict[str, Union[FeedAPIRecord, List[FeedAPIGroupRecord]]]]
:param to_sync: list of string feed names to sync metadata on
:type to_sync: List[str]
:param operation_id: UUID4 hexadecimal string
:type operation_id: Optional[str]
:param groups: whether or not to sync group metadata (defaults to True, which will sync group metadata)
:type groups: bool
:return: tuple, first element: dict of names mapped to db records post-sync only including records successfully updated by upstream, second element is a list of tuples where each tuple is (failed_feed_name, error_obj)
:rtype: Tuple[Dict[str, FeedMetadata], List[Tuple[str, Union[str, BaseException]]]
"""
if not to_sync:
return {}, []
db = get_session()
try:
logger.info(
"Syncing feed and group metadata from upstream source (operation_id={})".format(
operation_id
)
)
failed = []
db_feeds = MetadataSyncUtils._pivot_and_filter_feeds_by_config(
to_sync, list(source_feeds.keys()), get_all_feeds(db)
)
for feed_name, feed_api_record in source_feeds.items():
try:
logger.info(
"Syncing metadata for feed: {} (operation_id={})".format(
feed_name, operation_id
)
)
feed_metadata_map = MetadataSyncUtils._sync_feed_metadata(
db, feed_api_record, db_feeds, operation_id
)
if groups:
MetadataSyncUtils._sync_feed_group_metadata(
db, feed_api_record, feed_metadata_map, operation_id
)
except Exception as e:
logger.exception("Error syncing feed {}".format(feed_name))
logger.warn(
"Could not sync metadata for feed: {} (operation_id={})".format(
feed_name, operation_id
)
)
failed.append((feed_name, e))
finally:
db.flush()
# Reload
db_feeds = MetadataSyncUtils._pivot_and_filter_feeds_by_config(
to_sync, list(source_feeds.keys()), get_all_feeds(db)
)
db.commit()
logger.info(
"Metadata sync from feeds upstream source complete (operation_id={})".format(
operation_id
)
)
return db_feeds, failed
except Exception as e:
logger.error(
"Rolling back feed metadata update due to error: {} (operation_id={})".format(
e, operation_id
)
)
db.rollback()
raise
def call_api(
self,
method: callable,
path: str,
path_params=None,
query_params=None,
extra_headers=None,
body=None,
):
"""
Invoke the api
:param method: requests function to invoke (eg. requests.get)
:param path: url path
:param path_params: path params as dict
:param query_params: query param dict
:param extra_headers: header map to merge with default headers for this client
:param body: string body to send
:return:
"""
if path_params:
path_params = {
name: urlparse.quote(value)
for name, value in path_params.items()
}
final_url = urlparse.urljoin(self.url, path.format(**path_params))
else:
final_url = urlparse.urljoin(self.url, path)
request_headers = copy.copy(self.__headers__)
if extra_headers:
request_headers.update(extra_headers)
# Remove any None valued query params
if query_params:
filtered_qry_params = {
k: v
for k, v in filter(lambda x: x[1] is not None,
query_params.items())
}
else:
filtered_qry_params = None
logger.debug(
"Dispatching: url={url}, headers={headers}, body={body}, params={params}"
.format(
url=final_url,
headers=request_headers,
body=body[:512] +
("..." if len(body) > 512 else "") if body else body,
params=filtered_qry_params,
))
try:
if self.username and self.password:
return method(
url=final_url,
headers=request_headers,
data=body,
params=filtered_qry_params,
auth=(self.username, self.password),
)
else:
return method(
url=final_url,
headers=request_headers,
data=body,
params=filtered_qry_params,
)
except Exception as e:
logger.error("Failed client call to url: {}. Response: {}".format(
final_url, e.__dict__))
raise e
def init_oauth(app, grant_types, expiration_config):
"""
Configure the oauth routes and handlers via authlib
:return:
"""
logger.debug("Initializing oauth routes")
try:
tok_mgr = token_manager()
logger.info("Initialized the token manager")
except OauthNotConfiguredError:
logger.info("OAuth support not configured, cannot initialize it")
return None
except InvalidOauthConfigurationError:
logger.error("OAuth has invalid configuration, cannot initialize it")
raise
def query_client(client_id):
db = get_session()
c = db.query(OAuth2Client).filter_by(client_id=client_id).first()
return c
def do_not_save_token(token, request):
return None
# Don't use this (yet), due to token signing that allows system to verify without persistence
def save_token(token, request):
try:
if request.user:
user_id = request.user.username
else:
user_id = None
client = request.client
tok = OAuth2Token(client_id=client.client_id,
user_id=user_id,
**token)
db = get_session()
db.add(tok)
db.commit()
except:
logger.exception("Exception saving token")
raise
try:
# Initialize an anonymous client record
with session_scope() as db:
f = db.query(OAuth2Client).filter_by(client_id="anonymous").first()
if not f:
c = OAuth2Client()
c.client_id = "anonymous"
c.user_id = None
c.client_secret = None
c.issued_at = time.time() - 100
c.expires_at = time.time() + 1000
c.grant_type = "password"
c.token_endpoint_auth_method = "none"
c.client_name = "anonymous"
db.add(c)
except Exception as e:
logger.debug("Default client record init failed: {}".format(e))
app.config["OAUTH2_JWT_ENABLED"] = True
app.config["OAUTH2_ACCESS_TOKEN_GENERATOR"] = generate_token
app.config["OAUTH2_REFRESH_TOKEN_GENERATOR"] = False
# Only the password grant type is used, others can stay defaults
app.config["OAUTH2_TOKEN_EXPIRES_IN"] = expiration_config
app.config["OAUTH2_JWT_KEY"] = tok_mgr.default_issuer().signing_key
app.config["OAUTH2_JWT_ISS"] = tok_mgr.default_issuer().issuer
app.config["OAUTH2_JWT_ALG"] = tok_mgr.default_issuer().signing_alg
authz = AuthorizationServer(app,
query_client=query_client,
save_token=do_not_save_token)
# Support only the password grant for now
for grant in grant_types:
logger.debug("Registering oauth grant handler: {}".format(
getattr(grant, "GRANT_TYPE", "unknown")))
authz.register_grant(grant)
logger.debug("Oauth init complete")
return authz
def squash(unpackdir, cachedir, layers):
rootfsdir = unpackdir + "/rootfs"
if os.path.exists(unpackdir + "/squashed.tar"):
return (True)
whpatt = re.compile("\.wh\..*")
whopqpatt = re.compile("\.wh\.\.wh\.\.opq")
#slashprefixpatt = re.compile("^[\./|/]+")
slashprefixpatt = re.compile("^/+|\.+/+")
tarfiles = {}
tarfiles_members = {}
fhistory = {}
try:
logger.debug("Layers to process: {}".format(layers))
logger.debug("Pass 1: generating layer file timeline")
deferred_hardlinks_destination = {}
hardlink_destinations = {}
for l in layers:
htype, layer = l.split(":", 1)
layertar = get_layertarfile(unpackdir, cachedir, layer)
ltf = None
try:
lfhistory = {}
deferred_hardlinks = {}
logger.debug("processing layer {} - {}".format(l, layertar))
tarfiles[l] = tarfile.open(layertar,
mode='r',
format=tarfile.PAX_FORMAT)
tarfiles_members[l] = {}
for member in tarfiles[l].getmembers():
# clean up any prefix on the member names for history tracking purposes
tarfilename = member.name
member.name = slashprefixpatt.sub("", member.name)
if member.islnk() and member.linkname:
member.linkname = slashprefixpatt.sub(
"", member.linkname)
member.linkpath = member.linkname
member.pax_headers['path'] = member.name
# regular processing starts here
tarfiles_members[l][member.name] = member
filename = member.name
if filename not in lfhistory:
lfhistory[filename] = {}
lfhistory[filename]['latest_layer_tar'] = l
lfhistory[filename]['exists'] = True
if whopqpatt.match(os.path.basename(filename)):
# never include the wh itself
lfhistory[filename]['exists'] = False
# found an opq entry, which means that this files in the next layer down (only) should not be included
fsub = re.sub(r"\.wh\.\.wh\.\.opq", "", filename, 1)
fsub = re.sub("/+$", "", fsub)
for other_filename in fhistory.keys():
if re.match("^{}/".format(re.escape(fsub)),
other_filename):
if other_filename not in lfhistory:
lfhistory[other_filename] = {}
lfhistory[other_filename].update(
fhistory[other_filename])
lfhistory[other_filename]['exists'] = False
elif whpatt.match(os.path.basename(filename)):
# never include the wh itself
lfhistory[filename]['exists'] = False
fsub = re.sub(r"\.wh\.", "", filename, 1)
if fsub not in lfhistory:
lfhistory[fsub] = {}
if fsub in fhistory:
lfhistory[fsub].update(fhistory[fsub])
lfhistory[fsub]['exists'] = False
for other_filename in fhistory.keys():
if re.match("^{}/".format(re.escape(fsub)),
other_filename):
if other_filename not in lfhistory:
lfhistory[other_filename] = {}
lfhistory[other_filename].update(
fhistory[other_filename])
lfhistory[other_filename]['exists'] = False
if lfhistory[filename]['exists'] and member.islnk():
el = {
'hl_target_layer': l,
'hl_target_name': member.linkname,
'hl_replace': False,
}
lfhistory[filename].update(el)
if member.linkname not in hardlink_destinations:
hardlink_destinations[member.linkname] = []
el = {
'filename': filename,
'layer': l,
}
hardlink_destinations[member.linkname].append(el)
for filename in list(lfhistory.keys()):
if filename in hardlink_destinations:
for el in hardlink_destinations[filename]:
if el['layer'] != l:
if el['filename'] not in lfhistory:
lfhistory[el['filename']] = {}
lfhistory[el['filename']].update(
fhistory[el['filename']])
lfhistory[el['filename']]['hl_replace'] = True
fhistory.update(lfhistory)
except Exception as err:
logger.error(
"layer handler failure - exception: {}".format(err))
raise (err)
logger.debug("Pass 2: creating squashtar from layers")
allexcludes = []
with tarfile.open(os.path.join(unpackdir, "squashed.tar"),
mode='w',
format=tarfile.PAX_FORMAT) as oltf:
imageSize = 0
deferred_hardlinks = {}
for l in tarfiles_members.keys():
for filename in tarfiles_members[l].keys():
if fhistory[filename]['exists'] and fhistory[filename][
'latest_layer_tar'] == l:
member = tarfiles_members[l].get(filename)
if member.isreg():
memberfd = tarfiles[l].extractfile(member)
oltf.addfile(member, fileobj=memberfd)
elif member.islnk():
if fhistory[filename]['hl_replace']:
deferred_hardlinks[filename] = fhistory[
filename]
else:
oltf.addfile(member)
else:
oltf.addfile(member)
for filename in deferred_hardlinks.keys():
l = fhistory[filename]['latest_layer_tar']
member = tarfiles_members[l].get(filename)
logger.debug("deferred hardlink {}".format(fhistory[filename]))
try:
logger.debug(
"attempt to lookup deferred {} content source".format(
filename))
content_layer = fhistory[filename]['hl_target_layer']
content_filename = fhistory[filename]['hl_target_name']
logger.debug(
"attempt to extract deferred {} from layer {} (for lnk {})"
.format(content_filename, content_layer, filename))
content_member = tarfiles_members[content_layer].get(
content_filename)
content_memberfd = tarfiles[content_layer].extractfile(
content_member)
logger.debug(
"attempt to construct new member for deferred {}".
format(filename))
new_member = copy.deepcopy(content_member)
new_member.name = member.name
new_member.pax_headers['path'] = member.name
logger.debug(
"attempt to add final to squashed tar {} -> {}".format(
filename, new_member.name))
oltf.addfile(new_member, fileobj=content_memberfd)
except Exception as err:
import traceback
traceback.print_exc()
logger.warn(
"failed to store hardlink ({} -> {}) - exception: {}".
format(member.name, member.linkname, err))
finally:
logger.debug("Pass 3: closing layer tarfiles")
for l in tarfiles.keys():
if tarfiles[l]:
try:
tarfiles[l].close()
except Exception as err:
logger.error(
"failure closing tarfile {} - exception: {}".format(
l, err))
imageSize = 0
if os.path.exists(os.path.join(unpackdir, "squashed.tar")):
imageSize = os.path.getsize(os.path.join(unpackdir, "squashed.tar"))
return ("done", imageSize)
def upgrade(anchore_module, dontask, skip_db_compat_check):
"""
Run a Database Upgrade idempotently. If database is not initialized yet, but can be connected, then exit cleanly with status = 0, if no connection available then return error.
Otherwise, upgrade from the db running version to the code version and exit.
"""
ecode = 0
if not anchore_module:
module_name = "anchore_engine"
else:
module_name = str(anchore_module)
try:
try:
logger.info("Loading DB upgrade routines from module.")
module = importlib.import_module(module_name + ".db.entities.upgrade")
except Exception as err:
raise Exception("Input anchore-module (" + str(module_name) + ") cannot be found/imported - exception: " + str(err))
code_versions, db_versions = anchore_manager.cli.utils.init_database(upgrade_module=module, do_db_compatibility_check=(not skip_db_compat_check))
code_db_version = code_versions.get('db_version', None)
running_db_version = db_versions.get('db_version', None)
if not code_db_version or not running_db_version:
raise Exception("cannot get version information (code_db_version={} running_db_version={})".format(code_db_version, running_db_version))
elif code_db_version == running_db_version:
logger.info("Code and DB versions are in sync.")
ecode = 0
else:
logger.info("Detected anchore-engine version {}, running DB version {}.".format(code_db_version, running_db_version))
do_upgrade = False
if dontask:
do_upgrade = True
else:
try:
answer = raw_input("Performing this operation requires *all* anchore-engine services to be stopped - proceed? (y/N)")
except:
answer = "n"
if 'y' == answer.lower():
do_upgrade = True
if do_upgrade:
logger.info("Performing upgrade.")
try:
# perform the upgrade logic here
rc = module.run_upgrade()
if rc:
logger.info("Upgrade completed")
else:
logger.info("No upgrade necessary. Completed.")
except Exception as err:
raise err
else:
logger.info("Skipping upgrade.")
except Exception as err:
logger.error(anchore_manager.cli.utils.format_error_output(config, 'dbupgrade', {}, err))
if not ecode:
ecode = 2
anchore_manager.cli.utils.doexit(ecode)
def run_anchore_analyzers(staging_dirs, imageDigest, imageId, localconfig):
outputdir = staging_dirs['outputdir']
unpackdir = staging_dirs['unpackdir']
copydir = staging_dirs['copydir']
configdir = localconfig['service_dir']
# run analyzers
#anchore_module_root = resource_filename("anchore", "anchore-modules")
anchore_module_root = resource_filename("anchore_engine", "analyzers")
analyzer_root = os.path.join(anchore_module_root, "modules")
for f in list_analyzers():
#for f in os.listdir(analyzer_root):
# thecmd = os.path.join(analyzer_root, f)
# if re.match(".*\.py$", thecmd):
cmdstr = " ".join(
[f, configdir, imageId, unpackdir, outputdir, unpackdir])
if True:
try:
rc, sout, serr = utils.run_command(cmdstr)
sout = utils.ensure_str(sout)
serr = utils.ensure_str(serr)
if rc != 0:
raise Exception("command failed: cmd=" + str(cmdstr) +
" exitcode=" + str(rc) + " stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
else:
logger.debug("command succeeded: cmd=" + str(cmdstr) +
" stdout=" + str(sout).strip() + " stderr=" +
str(serr).strip())
except Exception as err:
logger.error("command failed with exception - " + str(err))
#raise err
analyzer_report = {}
for analyzer_output in os.listdir(
os.path.join(outputdir, "analyzer_output")):
if analyzer_output not in analyzer_report:
analyzer_report[analyzer_output] = {}
for analyzer_output_el in os.listdir(
os.path.join(outputdir, "analyzer_output", analyzer_output)):
if analyzer_output_el not in analyzer_report[analyzer_output]:
analyzer_report[analyzer_output][analyzer_output_el] = {
'base': {}
}
data = read_kvfile_todict(
os.path.join(outputdir, "analyzer_output", analyzer_output,
analyzer_output_el))
if data:
analyzer_report[analyzer_output][analyzer_output_el][
'base'] = read_kvfile_todict(
os.path.join(outputdir, "analyzer_output",
analyzer_output, analyzer_output_el))
else:
analyzer_report[analyzer_output].pop(analyzer_output_el, None)
if not analyzer_report[analyzer_output]:
analyzer_report.pop(analyzer_output, None)
return (analyzer_report)
def _get_system_user_credentials(self):
"""
Returns an AccessCredential object representing the system user
:return:
"""
cred = None
exp = None # Credential expiration, if needed
logger.debug("Loading system user creds")
with IdentityManager._cache_lock:
cached_cred = IdentityManager._credential_cache.lookup(
localconfig.SYSTEM_USERNAME)
if cached_cred is not None:
if cached_cred.is_expired():
# Flush it
logger.debug(
"Cached system credential is expired, flushing")
IdentityManager._credential_cache.delete(
localconfig.SYSTEM_USERNAME)
else:
logger.debug("Cached system credential still ok")
# Use it
cred = cached_cred
if cred is None:
logger.debug("Doing refresh/initial system cred load")
try:
tok_mgr = token_manager()
except OauthNotConfiguredError:
tok_mgr = None
# Generate one
if tok_mgr:
# Generate a token
usr = db_account_users.get(localconfig.SYSTEM_USERNAME,
session=self.session)
system_user_uuid = usr["uuid"]
tok, exp = tok_mgr.generate_token(system_user_uuid,
return_expiration=True)
logger.debug(
"Generated token with expiration {}".format(exp))
cred = HttpBearerCredential(tok, exp)
else:
rec = db_accounts.get(localconfig.SYSTEM_USERNAME,
session=self.session)
usr = db_account_users.get(localconfig.SYSTEM_USERNAME,
session=self.session)
if not rec or not usr:
logger.error(
"Could not find a system account or user. This is not an expected state"
)
raise Exception("No system account or user found")
# This will not work if the system admin has configured hashed passwords but not oauth. But, that should be caught at config validation.
cred = HttpBasicCredential(
usr["username"],
usr.get("credentials",
{}).get(UserAccessCredentialTypes.password,
{}).get("value"),
)
if cred is not None:
logger.debug("Caching system creds")
IdentityManager._credential_cache.cache_it(
localconfig.SYSTEM_USERNAME, cred)
return cred
def interactive_analyze(bodycontent):
try:
return_object = {}
httpcode = 500
request_inputs = anchore_engine.services.common.do_request_prep(
connexion.request, default_params={})
user_auth = request_inputs['auth']
method = request_inputs['method']
#bodycontent = request_inputs['bodycontent']
params = request_inputs['params']
userId = request_inputs['userId']
try:
#input prep
#jsondata = json.loads(bodycontent)
jsondata = bodycontent
tag = jsondata.pop('tag', None)
if not tag:
httpcode = 500
raise Exception("must supply a valid tag param in json body")
try:
# image prep
registry_creds = anchore_engine.clients.catalog.get_registry(
user_auth)
image_info = anchore_engine.services.common.get_image_info(
userId,
"docker",
tag,
registry_lookup=True,
registry_creds=registry_creds)
pullstring = image_info['registry'] + "/" + image_info[
'repo'] + "@" + image_info['digest']
fulltag = image_info['registry'] + "/" + image_info[
'repo'] + ":" + image_info['tag']
new_image_record = anchore_engine.services.common.make_image_record(
userId,
'docker',
fulltag,
registry_lookup=False,
registry_creds=(None, None))
image_detail = new_image_record['image_detail'][0]
if not image_detail:
raise Exception("no image found matching input")
except Exception as err:
httpcode = 404
raise Exception(str(err))
image_data, query_data = anchore_engine.services.analyzer.perform_analyze(
userId, pullstring, fulltag, image_detail, registry_creds)
if image_data:
return_object = image_data
httpcode = 200
else:
httpcode = 500
raise Exception("analyze resulted in empty analysis data")
except Exception as err:
logger.error(str(err))
raise err
except Exception as err:
logger.error(str(err))
return_object = str(err)
return (return_object, httpcode)
def _register(self):
if not self.is_enabled:
logger.error('Service not enabled in config, not registering service: ' + self.name)
raise Exception('No service enabled, cannot continue bootstrap')
logger.info('Registering service: {}'.format(self.name))
service_template = {
'type': 'anchore',
'base_url': 'N/A',
'status_base_url': 'N/A',
'version': 'v1',
'short_description': ''
}
hstring = 'http'
if 'external_tls' in self.configuration:
if self.configuration.get('external_tls', False):
hstring = 'https'
elif 'ssl_enable' in self.configuration:
if self.configuration.get('ssl_enable', False):
hstring = 'https'
endpoint_hostname = endpoint_port = endpoint_hostport = None
if self.configuration.get('external_hostname', False):
endpoint_hostname = self.configuration.get('external_hostname')
elif self.configuration.get('endpoint_hostname', False):
endpoint_hostname = self.configuration.get('endpoint_hostname')
if self.configuration.get('external_port', False):
endpoint_port = int(self.configuration.get('external_port'))
elif self.configuration.get('port', False):
endpoint_port = int(self.configuration.get('port'))
if endpoint_hostname:
endpoint_hostport = endpoint_hostname
if endpoint_port:
endpoint_hostport = endpoint_hostport + ":" + str(endpoint_port)
if endpoint_hostport:
service_template['base_url'] = "{}://{}".format(hstring, endpoint_hostport)
else:
raise Exception("could not construct service base_url - please check service configuration for hostname/port settings")
try:
service_template['status'] = False
service_template['status_message'] = taskstate.base_state('service_status')
with session_scope() as dbsession:
service_records = db_services.get_byname(self.__service_name__, session=dbsession)
# fail if trying to add a service that must be unique in the system, but one already is registered in DB
if self.__is_unique_service__:
if len(service_records) > 1:
raise Exception('more than one entry for service type (' + str(
self.__service_name__) + ') exists in DB, but service must be unique - manual DB intervention required')
for service_record in service_records:
if service_record and (service_record['hostid'] != self.instance_id):
raise Exception('service type (' + str(self.__service_name__) + ') already exists in system with different host_id - detail: my_host_id=' + str(
self.instance_id) + ' db_host_id=' + str(service_record['hostid']))
# if all checks out, then add/update the registration
ret = db_services.add(self.instance_id, self.__service_name__, service_template, session=dbsession)
try:
my_service_record = {
'hostid': self.instance_id,
'servicename': self.__service_name__,
}
my_service_record.update(service_template)
servicestatus.set_my_service_record(my_service_record)
self.service_record = my_service_record
except Exception as err:
logger.warn('could not set local service information - exception: {}'.format(str(err)))
except Exception as err:
raise err
service_record = servicestatus.get_my_service_record()
servicestatus.set_status(service_record, up=True, available=True, update_db=True, versions=self.versions)
logger.info('Service registration complete')
return True
def squash(unpackdir, cachedir, layers):
rootfsdir = unpackdir + "/rootfs"
if os.path.exists(unpackdir + "/squashed.tar"):
return (True)
if not os.path.exists(rootfsdir):
os.makedirs(rootfsdir)
revlayer = list(layers)
revlayer.reverse()
l_excludes = {}
l_opqexcludes = {
} # stores list of special files to exclude only for next layer (.wh..wh..opq handling)
last_opqexcludes = {} # opq exlcudes for the last layer
for l in revlayer:
htype, layer = l.split(":", 1)
layertar = get_layertarfile(unpackdir, cachedir, layer)
count = 0
logger.debug("\tPass 1: " + str(layertar))
whpatt = re.compile(".*/\.wh\..*")
whopqpatt = re.compile(".*/\.wh\.\.wh\.\.opq")
l_opqexcludes[layer] = {}
myexcludes = {}
opqexcludes = {}
tarfilenames = get_tar_filenames(layertar)
for fname in tarfilenames:
# checks for whiteout conditions
if whopqpatt.match(fname):
# found an opq entry, which means that this files in the next layer down (only) should not be included
fsub = re.sub(r"\.wh\.\.wh\.\.opq", "", fname, 1)
# never include the whiteout file itself
myexcludes[fname] = True
opqexcludes[fsub] = True
elif whpatt.match(fname):
# found a normal whiteout, which means that this file in any lower layer should be excluded
fsub = re.sub(r"\.wh\.", "", fname, 1)
# never include a whiteout file
myexcludes[fname] = True
myexcludes[fsub] = True
else:
# if the last processed layer had an opq whiteout, check file to see if it lives in the opq directory
if last_opqexcludes:
dtoks = fname.split("/")
for i in range(0, len(dtoks)):
dtok = '/'.join(dtoks[0:i])
dtokwtrail = '/'.join(dtoks[0:i]) + "/"
if dtok in last_opqexcludes or dtokwtrail in last_opqexcludes:
l_opqexcludes[layer][fname] = True
break
# build up the list of excludes as we move down the layers
for l in l_excludes.keys():
myexcludes.update(l_excludes[l])
l_excludes[layer] = myexcludes
last_opqexcludes.update(opqexcludes)
logger.debug("Pass 3: untarring layers with exclusions")
imageSize = 0
for l in layers:
htype, layer = l.split(":", 1)
layertar = get_layertarfile(unpackdir, cachedir, layer)
imageSize = imageSize + os.path.getsize(layertar)
# write out the exluded files, adding the per-layer excludes if present
with open(unpackdir + "/efile", 'w') as OFH:
for efile in l_excludes[layer]:
OFH.write("%s\n" % efile)
if layer in l_opqexcludes and l_opqexcludes[layer]:
for efile in l_opqexcludes[layer]:
logger.debug("adding special for layer exclude: " +
str(efile))
OFH.write("%s\n" % efile)
retry = True
success = False
last_err = None
max_retries = 10
retries = 0
while (not success) and (retry):
tarcmd = "tar -C " + rootfsdir + " -x -X " + unpackdir + "/efile -f " + layertar
logger.debug("untarring squashed tarball: " + str(tarcmd))
try:
rc, sout, serr = utils.run_command(tarcmd)
if rc != 0:
logger.debug("tar error encountered, attempting to handle")
handled = handle_tar_error(tarcmd,
rc,
sout,
serr,
unpackdir=unpackdir,
rootfsdir=rootfsdir,
layer=layer,
layertar=layertar)
if not handled:
raise Exception("command failed: cmd=" + str(tarcmd) +
" exitcode=" + str(rc) + " stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
else:
logger.debug(
"tar error successfully handled, retrying")
else:
logger.debug("command succeeded: stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
success = True
except Exception as err:
logger.error("command failed with exception - " + str(err))
last_err = err
success = False
retry = False
# safety net
if retries > max_retries:
retry = False
retries = retries + 1
if not success:
if last_err:
raise last_err
else:
raise Exception("unknown exception in untar")
return ("done", imageSize)
def _sync_group(self,
group_download_result: GroupDownloadResult,
full_flush=False,
local_repo=None,
operation_id=None):
"""
Sync data from a single group and return the data. This operation is scoped to a transaction on the db.
:param group_download_result
:return:
"""
total_updated_count = 0
result = build_group_sync_result()
result['group'] = group_download_result.group
sync_started = None
db = get_session()
db.refresh(self.metadata)
group_db_obj = self.group_by_name(group_download_result.group)
if not group_db_obj:
logger.error(
log_msg_ctx(
operation_id, group_download_result.feed,
group_download_result.group,
'Skipping group sync. Record not found in db, should have been synced already'
))
return result
sync_started = time.time()
download_started = group_download_result.started.replace(
tzinfo=datetime.timezone.utc)
try:
updated_images = set(
) # To get unique set of all images updated by this sync
if full_flush:
logger.info(
log_msg_ctx(operation_id, group_download_result.feed,
group_download_result.group,
'Performing group data flush prior to sync'))
self._flush_group(group_db_obj, operation_id=operation_id)
mapper = self._load_mapper(group_db_obj)
# Iterate thru the records and commit
count = 0
for record in local_repo.read(group_download_result.feed,
group_download_result.group, 0):
mapped = mapper.map(record)
updated_image_ids = self.update_vulnerability(
db,
mapped,
vulnerability_processing_fn=VulnerabilityFeed.
__vuln_processing_fn__)
updated_images = updated_images.union(
set(updated_image_ids
)) # Record after commit to ensure in-sync.
merged = db.merge(mapped)
total_updated_count += 1
count += 1
if len(updated_image_ids) > 0:
db.flush(
) # Flush after every one so that mem footprint stays small if lots of images are updated
if count >= self.RECORDS_PER_CHUNK:
# Commit
group_db_obj.count = self.record_count(
group_db_obj.name, db)
db.commit()
logger.info(
log_msg_ctx(
operation_id, group_download_result.feed,
group_download_result.group,
'DB Update Progress: {}/{}'.format(
total_updated_count,
group_download_result.total_records)))
db = get_session()
count = 0
else:
group_db_obj.count = self.record_count(group_db_obj.name, db)
db.commit()
logger.info(
log_msg_ctx(
operation_id, group_download_result.feed,
group_download_result.group,
'DB Update Progress: {}/{}'.format(
total_updated_count,
group_download_result.total_records)))
db = get_session()
logger.debug(
log_msg_ctx(
operation_id, group_download_result.feed,
group_download_result.group,
'Updating last sync timestamp to {}'.format(
download_started)))
group_db_obj = self.group_by_name(group_download_result.group)
group_db_obj.last_sync = download_started
group_db_obj.count = self.record_count(group_db_obj.name, db)
db.add(group_db_obj)
db.commit()
except Exception as e:
logger.exception(
log_msg_ctx(operation_id, group_download_result.feed,
group_download_result.group,
'Error syncing group'))
db.rollback()
raise e
finally:
total_group_time = time.time() - download_started.timestamp()
sync_time = time.time() - sync_started
logger.info(
log_msg_ctx(operation_id, group_download_result.feed,
group_download_result.group,
'Sync to db duration: {} sec'.format(sync_time)))
logger.info(
log_msg_ctx(
operation_id, group_download_result.feed,
group_download_result.group,
'Total sync, including download, duration: {} sec'.format(
total_group_time)))
result['updated_record_count'] = total_updated_count
result['status'] = 'success'
result['total_time_seconds'] = total_group_time
result['updated_image_count'] = 0
return result
