def load_config(configdir=None, configfile=None, validate_params=None): global localconfig load_defaults(configdir=configdir) if not configfile: configfile = os.path.join(localconfig["service_dir"], DEFAULT_CONFIG_FILENAME) if not os.path.exists(configfile): raise Exception("config file (" + str(configfile) + ") not found") else: try: confdata = read_config(configfile=configfile) update_merge(localconfig, confdata) except Exception as err: raise err try: validate_config(localconfig, validate_params=validate_params) except Exception as err: raise Exception("invalid configuration: details - " + str(err)) # setup service dir if not os.path.exists(os.path.join(localconfig["service_dir"])): success = False for i in range(0, 5): try: os.makedirs(os.path.join(localconfig["service_dir"])) success = True except: time.sleep(1) if not success: raise Exception("could not create service directory: " + str(localconfig["service_dir"])) # setup tmp dir if not os.path.exists(os.path.join(localconfig["tmp_dir"])): success = False for i in range(0, 5): try: os.makedirs(os.path.join(localconfig["tmp_dir"])) success = True except: time.sleep(1) if not success: raise Exception("could not create temporary directory: " + str(localconfig["tmp_dir"])) # copy the src installed files unless they already exist in the service dir conf load_policy_bundle_paths() load_filepath_to_config("anchore_scanner_analyzer_config_file", "analyzer_config.yaml") # generate/setup the host_id in the service_dir localconfig["host_id"] = get_host_id() # any special deployment/environment specific config handling here, via extension config localconfig["image_content_types"] = image_content_types localconfig["image_metadata_types"] = image_metadata_types ext_config = {} for mod in "anchore_engine", "anchore_enterprise": try: ext_config_file = os.path.join(resource_filename(mod, "conf/"), "extensions.yaml") except Exception as err: logger.debug( "skipping config extension load for module {} - exception: {}". format(mod, err)) ext_config_file = None if ext_config_file and os.path.exists(ext_config_file): try: with open(ext_config_file, "r") as FH: d = yaml.safe_load(FH) if d: ext_config.update(d) except Exception as err: logger.error( "failed to load extensions.yaml - exception: {}".format( err)) if ext_config: if ext_config.get("content_types", []): localconfig["image_content_types"].extend( ext_config.get("content_types")) if ext_config.get("metadata_types", []): localconfig["image_metadata_types"].extend( ext_config.get("metadata_types")) analyzer_config = localconfig.get("services", {}).get("analyzer", {}) if analyzer_config: localconfig["services"]["analyzer"]["analyzer_driver"] = "nodocker" return localconfig
def exec(docker_archive, anchore_archive, digest, parent_digest, image_id, tag, account_id, manifest, dockerfile, created_at, annotation): """ Analyze a local image stored as a docker archive (output result of 'docker save'), and generate an anchore image archive tarball ready for import into an anchore engine. DOCKER_ARCHIVE : Location of input docker archive tarfile to analyze ANCHORE_ARCHIVE : Location of output anchore image archive to write """ global config # this could be improved to allow use to input timestamps (created_at, analyzed_at, etc) now = int(time.time()) try: try: imageDigest = None manifest_data = None rawmanifest = None if (not manifest and not digest) or (manifest and digest): raise Exception( "must supply either an image digest or a valid manifest, but not both" ) if os.path.exists(anchore_archive): raise Exception( "the supplied anchore archive file ({}) already exists, please remove and try again" .format(anchore_archive)) if manifest: try: with open(manifest, 'r') as FH: # TODO implement manifest validator for anchore requirements, specifically rawmanifest = FH.read() input_manifest_data = json.loads(rawmanifest) imageDigest = manifest_to_digest(rawmanifest) except Exception as err: raise ValueError( "cannot calculate digest from supplied manifest - exception: {}" .format(err)) if not re.match("^sha256:[\d|a-f]{64}$", imageDigest): raise ValueError( "input digest does not validate - must be sha256:<64 hex characters>" ) if parent_digest: if re.match("^sha256:[\d|a-f]{64}$", parent_digest): parentDigest = parent_digest else: raise ValueError("input parent_digest does not validate") else: parentDigest = imageDigest if image_id: if re.match("^[\d|a-f]{64}$", image_id): imageId = image_id else: raise ValueError("input image_id does not validate") else: # TODO this could be improved to generate imageId from configuration hash imageId = "{}".format(''.join( [random.choice('0123456789abcdef') for x in range(0, 64)])) if account_id: userId = account_id else: userId = 'admin' if created_at: try: if int(created_at) < 0 or int(created_at) > now + 1: raise Exception() except Exception as err: raise ValueError( "created_at must by a unix timestamp between 0 and now ({})" .format(now)) else: created_at = now try: inputTag = tag image_info = parse_dockerimage_string(inputTag) fulltag = "{}/{}:{}".format(image_info['registry'], image_info['repo'], image_info['tag']) fulldigest = "{}/{}@{}".format(image_info['registry'], image_info['repo'], imageDigest) except Exception as err: raise ValueError( "input tag does not validate - exception: {}".format(err)) dockerfile_mode = "Guessed" dockerfile_contents = None if dockerfile: with open(dockerfile, 'r') as FH: dockerfile_contents = ensure_str( base64.b64encode(ensure_bytes(FH.read()))) dockerfile_mode = "Actual" annotations = {} if annotation: for a in annotation: try: (k, v) = a.split('=', 1) if k and v: annotations[k] = v else: raise Exception("found null in key or value") except Exception as err: raise ValueError( "annotation format error - annotations must be of the form (--annotation key=value), found: {}" .format(a)) workspace_root = config['tmp_dir'] except Exception as err: # input setup/validation failure raise err logger.debug( "input has been prepared: imageDigest={} parentDigest={} imageId={} inputTag={} fulltag={} fulldigest={} userId={} annotations={} created_at={}" .format(imageDigest, parentDigest, imageId, inputTag, fulltag, fulldigest, userId, annotations, created_at)) # create an image record try: image_record = make_image_record(userId, 'docker', None, image_metadata={ 'tag': fulltag, 'digest': fulldigest, 'imageId': imageId, 'parentdigest': parentDigest, 'created_at': created_at, 'dockerfile': dockerfile_contents, 'dockerfile_mode': dockerfile_mode, 'annotations': annotations }, registry_lookup=False, registry_creds=(None, None)) image_record['created_at'] = created_at image_record['last_updated'] = created_at image_record['analyzed_at'] = now image_record['analysis_status'] = 'analyzed' image_record['image_status'] = 'active' image_record['record_state_key'] = 'active' for image_detail in image_record['image_detail']: image_detail['created_at'] = created_at image_detail['last_updated'] = created_at image_detail['tag_detected_at'] = created_at image_detail['record_state_key'] = 'active' except Exception as err: # image record setup fail raise err # perform analysis try: image_data, analyzed_manifest_data = analyze_image( userId, rawmanifest, image_record, workspace_root, config, registry_creds=[], use_cache_dir=None, image_source='docker-archive', image_source_meta=docker_archive) image_content_data = {} for content_type in anchore_engine.common.image_content_types + anchore_engine.common.image_metadata_types: try: image_content_data[ content_type] = anchore_engine.common.helpers.extract_analyzer_content( image_data, content_type, manifest=input_manifest_data) except: image_content_data[content_type] = {} anchore_engine.common.helpers.update_image_record_with_analysis_data( image_record, image_data) image_record['image_size'] = int(image_record['image_size']) except Exception as err: # image analysis fail raise err # generate an output image archive tarball archive_file = anchore_archive try: with ImageArchive.for_writing(archive_file) as img_archive: img_archive.account = userId img_archive.image_digest = imageDigest img_archive.manifest.metadata = { 'versions': localconfig.get_versions(), 'image_id': imageId, 'image_record': json.dumps(image_record, sort_keys=True) } pack_data = {'document': image_data} data = ensure_bytes(json.dumps(pack_data, sort_keys=True)) img_archive.add_artifact('analysis', source=ObjectStoreLocation( bucket='analysis_data', key=imageDigest), data=data, metadata=None) pack_data = {'document': image_content_data} data = ensure_bytes(json.dumps(pack_data, sort_keys=True)) img_archive.add_artifact('image_content', source=ObjectStoreLocation( bucket='image_content_data', key=imageDigest), data=data, metadata=None) pack_data = {'document': input_manifest_data} data = ensure_bytes(json.dumps(pack_data, sort_keys=True)) img_archive.add_artifact('image_manifest', source=ObjectStoreLocation( bucket='manifest_data', key=imageDigest), data=data, metadata=None) except Exception as err: # archive tarball generate fail raise err except Exception as err: logger.error( anchore_manager.cli.utils.format_error_output( click_config, 'db', {}, err)) sys.exit(2) click.echo( "Analysis complete for image {} - archive file is located at {}". format(imageDigest, archive_file))
def validate_config(config, validate_params=None): """ Validate the configuration with required keys and values :param config: the config dict to validate :param validate_params: dict of top level config properties and boolean flag :return: true if passes validation, false otherwise """ ret = True if validate_params is None: validate_params = default_required_config_params try: # ensure there aren't any left over unset variables confbuf = json.dumps(config) patt = re.match(".*(\${ANCHORE.*?}).*", confbuf, re.DOTALL) if patt: raise Exception( "variable overrides found in configuration file that are unset (" + str(patt.group(1)) + ")") # top level checks if 'services' in validate_params and validate_params['services']: if 'services' not in config or not config['services']: raise Exception( "no 'services' definition in configuration file") else: for k in list(config['services'].keys()): if not config['services'][k] or 'enabled' not in config[ 'services'][k]: raise Exception( "service (" + str(k) + ") defined, but no values are specified (need at least 'enabled: <True|False>')" ) else: service_config = config['services'][k] # check to ensure the listen/port/endpoint_hostname params are set for all services check_keys = ['endpoint_hostname', 'listen', 'port'] for check_key in check_keys: if check_key not in service_config: raise Exception( "the following values '{}' must be set for all services, but service '{}' does not have them set (missing '{}')" .format(check_keys, k, check_key)) # check to ensure that if any TLS params are set, then they all must be set found_key = 0 check_keys = ['ssl_enable', 'ssl_cert', 'ssl_key'] for check_key in check_keys: if check_key in service_config: found_key = found_key + 1 if found_key != 0 and found_key != 3: raise Exception( "if any one of (" + ','.join(check_keys) + ") are specified, then all must be specified for service '" + str(k) + "'") if 'credentials' in validate_params and validate_params['credentials']: if 'credentials' not in config or not config['credentials']: raise Exception( "no 'credentials' definition in configuration file") else: credentials = config['credentials'] for check_key in ['database']: if check_key not in credentials: raise Exception( "no '" + str(check_key) + "' definition in 'credentials' section of configuration file" ) elif not credentials[check_key]: raise Exception( "'" + str(check_key) + "' is in configuration file, but is empty (has no records)" ) # database checks for check_key in ['db_connect', 'db_connect_args']: if check_key not in credentials['database']: raise Exception( "no '" + str(check_key) + "' definition in 'credentials'/'database' section of configuration file" ) # webhook checks if 'webhooks' in validate_params and validate_params['webhooks']: if 'webhooks' not in config or not config['webhooks']: logger.warn( "no webhooks defined in configuration file - notifications will be disabled" ) if 'user_authentication' in validate_params and validate_params[ 'user_authentication']: validate_user_auth_config(config) validate_key_config(config, required=False) except Exception as err: logger.error(str(err)) raise err # raise Exception("TEST") return (ret)
def get_client(feeds_url=None, user=None, conn_timeout=None, read_timeout=None, ssl_verify=None): """ Returns a configured client based on the local config. Reads configuration from the loaded system configuration. Uses the admin user's credentials for the feed service if they are available in the external_service_auths/anchoreio/anchorecli/auth json path of the config file. If no specific user credentials are found then the anonymous user credentials are used. :return: initialize AnchoreIOFeedClient """ logger.debug( "Initializing a feeds client: url={}, user={}, conn_timeout={}, read_timeout={}" .format( feeds_url, user if user is None or type(user) not in [tuple, list] or len(user) == 0 else (user[0], "***redacted**"), conn_timeout, read_timeout, )) if not (feeds_url and user and conn_timeout and read_timeout): conf = localconfig.get_config() if not conf: logger.error( "No configuration available. Cannot initialize feed client") raise ValueError("None for local config") else: conf = { "feeds": { "connection_timeout_seconds": conn_timeout, "read_timeout_seconds": read_timeout, "url": feeds_url, "ssl_verify": ssl_verify, } } if not conn_timeout: conn_timeout = conf.get("feeds", {}).get("connection_timeout_seconds") if not read_timeout: read_timeout = conf.get("feeds", {}).get("read_timeout_seconds") if not feeds_url: feeds_url = conf.get("feeds", {}).get("url") if not feeds_url: raise ValueError("no feed service url available") verify = conf.get("feeds", {}).get("ssl_verify", True) password = None if not user: try: admin_usr = (conf.get("credentials", {}).get("users", {}).get( "admin", {}).get("external_service_auths", {}).get("anchoreio", {}).get("anchorecli", {}).get("auth")) if admin_usr: user, password = admin_usr.split(":") except AttributeError: # Something isn't found or was set to None. pass else: user, password = user[0], user[1] if not user: user = conf.get("feeds", {}).get("anonymous_user_username") password = conf.get("feeds", {}).get("anonymous_user_password") logger.debug("using values: " + str([feeds_url, user, conn_timeout, read_timeout])) http_client = HTTPBasicAuthClient( username=user, password=password, connect_timeout=conn_timeout, read_timeout=read_timeout, verify=verify, ) return FeedServiceClient(endpoint=feeds_url, http_client=http_client)
def get_image_metadata_v1(staging_dirs, imageDigest, imageId, manifest_data, dockerfile_contents="", dockerfile_mode=""): outputdir = staging_dirs['outputdir'] unpackdir = staging_dirs['unpackdir'] copydir = staging_dirs['copydir'] docker_history = [] layers = [] dockerfile_mode = "Guessed" dockerfile_contents = dockerfile_contents imageArch = "" try: imageArch = manifest_data['architecture'] except: imageArch = "" try: for fslayer in manifest_data['fsLayers']: layers.append(fslayer['blobSum']) except Exception as err: logger.error("cannot get layers - exception: " + str(err)) raise err try: hfinal = [] count = 0 for rawhel in manifest_data['history']: hel = json.loads(rawhel['v1Compatibility']) try: lsize = hel['Size'] except: lsize = 0 if hel['container_config']['Cmd']: lcreatedby = ' '.join(hel['container_config']['Cmd']) else: lcreatedby = "" lcreated = hel['created'] lid = layers[count] count = count + 1 hfinal.append({ 'Created': lcreated, 'CreatedBy': lcreatedby, 'Comment': '', 'Id': lid, 'Size': lsize, 'Tags': [] }) docker_history = hfinal if hfinal: with open(os.path.join(unpackdir, "docker_history.json"), 'w') as OFH: OFH.write(json.dumps(hfinal)) except Exception as err: logger.error("cannot construct history - exception: " + str(err)) raise err if not dockerfile_contents: # get dockerfile_contents (translate history to guessed DF) # TODO 'FROM' guess? dockerfile_contents = "FROM scratch\n" for hel in docker_history: patt = re.match("^/bin/sh -c #\(nop\) +(.*)", hel['CreatedBy']) if patt: cmd = patt.group(1) elif hel['CreatedBy']: cmd = "RUN " + hel['CreatedBy'] else: cmd = None if cmd: dockerfile_contents = dockerfile_contents + cmd + "\n" dockerfile_mode = "Guessed" elif not dockerfile_mode: dockerfile_mode = "Actual" layers.reverse() return (docker_history, layers, dockerfile_contents, dockerfile_mode, imageArch)
def download_image(fulltag, copydir, user=None, pw=None, verify=True, manifest=None, use_cache_dir=None, dest_type='oci'): try: proc_env = os.environ.copy() if user and pw: proc_env['SKOPUSER'] = user proc_env['SKOPPASS'] = pw credstr = '--src-creds \"${SKOPUSER}\":\"${SKOPPASS}\"' else: credstr = "" if verify: tlsverifystr = "--src-tls-verify=true" else: tlsverifystr = "--src-tls-verify=false" if use_cache_dir and os.path.exists(use_cache_dir): cachestr = "--dest-shared-blob-dir " + use_cache_dir else: cachestr = "" if dest_type == 'oci': if manifest: with open(os.path.join(copydir, "manifest.json"), 'w') as OFH: OFH.write(manifest) cmd = [ "/bin/sh", "-c", "skopeo copy {} {} {} docker://{} oci:{}:image".format( tlsverifystr, credstr, cachestr, fulltag, copydir) ] else: cmd = [ "/bin/sh", "-c", "skopeo copy {} {} docker://{} dir:{}".format( tlsverifystr, credstr, fulltag, copydir) ] cmdstr = ' '.join(cmd) try: rc, sout, serr = anchore_engine.services.common.run_command_list( cmd, env=proc_env) if rc != 0: raise Exception("command failed: cmd=" + str(cmdstr) + " exitcode=" + str(rc) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err except Exception as err: raise err return (True)
def get_image_manifest(userId, image_info, registry_creds): logger.debug("get_image_manifest input: " + str(userId) + " : " + str(image_info) + " : " + str(time.time())) user = pw = None repo = url = None registry_verify = True registry = image_info['registry'] try: user, pw, registry_verify = anchore_engine.auth.common.get_creds_by_registry( registry, registry_creds=registry_creds) except Exception as err: raise err if registry == 'docker.io': url = "https://index.docker.io" if not re.match(".*/.*", image_info['repo']): repo = "library/" + image_info['repo'] else: repo = image_info['repo'] else: url = "https://" + registry repo = image_info['repo'] if image_info['digest']: tag = None input_digest = image_info['digest'] fulltag = "{}/{}@{}".format(registry, repo, input_digest) else: input_digest = None tag = image_info['tag'] fulltag = "{}/{}:{}".format(registry, repo, tag) manifest = digest = None logger.debug("trying to get manifest/digest for image (" + str(fulltag) + ")") err = None try: if tag: manifest, digest = get_image_manifest_skopeo( url, registry, repo, intag=tag, user=user, pw=pw, verify=registry_verify) elif input_digest: manifest, digest = get_image_manifest_skopeo( url, registry, repo, indigest=input_digest, user=user, pw=pw, verify=registry_verify) else: raise Exception("neither tag nor digest was given as input") except Exception as err: logger.error("could not fetch manifest/digest: " + str(err)) manifest = digest = None if manifest and digest: return (manifest, digest) logger.error( "could not get manifest/digest for image ({}) from registry ({}) - error: {}" .format(fulltag, url, err)) raise Exception( "could not get manifest/digest for image ({}) from registry ({}) - error: {}" .format(fulltag, url, err)) return ({}, "")
def lookup_registry_image(self, tag=None, digest=None): if not tag and not digest: logger.error("no input (tag=, digest=)") raise Exception("bad input") return self.call_api(http.anchy_get, 'registry_lookup', query_params={'digest': digest, 'tag': tag})
def get_image_manifest_orig(userId, image_info, registry_creds): logger.debug("get_image_manifest input: " + str(userId) + " : " + str(image_info) + " : " + str(time.time())) user = pw = None registry_verify = True registry = image_info['registry'] try: user, pw, registry_verify = anchore_engine.auth.common.get_creds_by_registry( registry, registry_creds=registry_creds) except Exception as err: raise err if False: # replaced with utility func above try: for registry_record in registry_creds: if registry_record['registry'] == registry: if registry_record['record_state_key'] not in ['active']: try: last_try = int(registry_record['record_state_val']) except: last_try = 0 if (int(time.time()) - last_try) < 60: logger.debug( "SKIPPING REGISTRY ATTEMPT: " + str(registry_record['record_state_key'])) raise Exception( "registry not available - " + str(registry_record['record_state_key'])) user, pw = anchore_engine.auth.common.get_docker_registry_userpw( registry_record) registry_verify = registry_record['registry_verify'] break except Exception as err: raise err if registry == 'docker.io': url = "https://index.docker.io" if not re.match(".*/.*", image_info['repo']): repo = "library/" + image_info['repo'] else: repo = image_info['repo'] else: url = "https://" + registry repo = image_info['repo'] oauth_err = basicauth_err = skopeoauth_err = "N/A" if image_info['digest']: pullstring = image_info['digest'] else: pullstring = image_info['tag'] auth_funcs = [ get_image_manifest_oauth2, get_image_manifest_docker_registry, get_image_manifest_skopeo ] auth_errors = {} manifest = digest = None for af in auth_funcs: manifest = digest = None try: imagestr = url + "/" + repo + ":" + pullstring except: imagestr = pullstring logger.debug("trying to get manifest/digest for image (" + str(imagestr) + ") using (" + str(af.__name__) + ")") try: manifest, digest = af(url, registry, repo, pullstring, user=user, pw=pw, verify=registry_verify) except Exception as err: logger.debug("could not get manifest/digest for image (" + imagestr + ") using (" + str(af.__name__) + ") - exception: " + str(err)) auth_errors[af.__name__] = str(err) if manifest and digest: break if manifest and digest: return (manifest, digest) logger.error( "could not get manifest/digest for image using any auth method: (" + str(pullstring) + "): " + str(auth_errors)) raise Exception( "could not get manifest/digest for image using any auth method: (" + str(pullstring) + "): " + str(auth_errors)) return ({}, "")
def handle_image_analyzer(*args, **kwargs): global system_user_auth, queuename, servicename cycle_timer = kwargs['mythread']['cycle_timer'] localconfig = anchore_engine.configuration.localconfig.get_config() system_user_auth = localconfig['system_user_auth'] threads = [] layer_cache_dirty = True while (True): logger.debug("analyzer thread cycle start") try: myconfig = localconfig['services']['analyzer'] max_analyze_threads = int(myconfig.get('max_threads', 1)) layer_cache_enable = myconfig.get('layer_cache_enable', False) logger.debug("max threads: " + str(max_analyze_threads)) if len(threads) < max_analyze_threads: logger.debug("analyzer has free worker threads {} / {}".format( len(threads), max_analyze_threads)) qobj = simplequeue.dequeue(system_user_auth, queuename) if qobj: logger.debug("got work from queue task Id: {}".format( qobj.get('queueId', 'unknown'))) myqobj = copy.deepcopy(qobj) logger.spew("incoming queue object: " + str(myqobj)) logger.debug("incoming queue task: " + str(myqobj.keys())) logger.debug("starting thread") athread = threading.Thread(target=process_analyzer_job, args=(system_user_auth, myqobj, layer_cache_enable)) athread.start() threads.append(athread) logger.debug("thread started") layer_cache_dirty = True else: logger.debug( "analyzer queue is empty - no work this cycle") else: logger.debug("all workers are busy") alive_threads = [] while (threads): athread = threads.pop() if not athread.isAlive(): try: logger.debug("thread completed - joining") athread.join() logger.debug("thread joined") except Exception as err: logger.warn("cannot join thread - exception: " + str(err)) else: alive_threads.append(athread) threads = alive_threads if layer_cache_enable and layer_cache_dirty and len(threads) == 0: logger.debug("running layer cache handler") try: handle_layer_cache() layer_cache_dirty = False except Exception as err: logger.warn("layer cache management failed - exception: " + str(err)) except Exception as err: import traceback traceback.print_exc() logger.error(str(err)) logger.debug("analyzer thread cycle complete: next in " + str(cycle_timer)) time.sleep(cycle_timer) return (True)
def notify(self, notification_type, notification_value): """ No-Op for the default handler since permissions are ephemeral. :param notification_type: :param notification_value: :return: """ logger.info('Calling notification!') retries = 3 try: if not ExternalAuthzRealm.__client__: logger.warn( 'Got authz notification type: {} value:{}, but no client configured so nothing to do' .format(notification_type, notification_value)) return True else: if NotificationTypes.domain_created == notification_type: fn = ExternalAuthzRealm.__client__.initialize_domain elif NotificationTypes.domain_deleted == notification_type: fn = ExternalAuthzRealm.__client__.delete_domain elif NotificationTypes.principal_created == notification_type: fn = ExternalAuthzRealm.__client__.initialize_principal elif NotificationTypes.principal_deleted == notification_type: fn = ExternalAuthzRealm.__client__.delete_principal else: fn = None if fn is None: logger.warn( 'Got notification type {} with no handler mapped'.format( notification_type)) return err = None for i in range(retries): try: resp = fn(notification_value) if not resp: logger.warn( 'Bad response from authz service, will retry: {}'. format(resp)) else: logger.debug( 'Notification succeeded to authz plugin service') break except Exception as ex: err = ex logger.exception( 'Error calling {} against authz plugin client'.format( fn.__name__)) else: logger.error( 'Could not confirm successful response of authz handler for notification {} with value {}' .format(notification_type, notification_value)) raise Exception( 'Error invoking POST /domains on external authz handler: {}' .format( str(err) if err else 'Retry count exceeded {}'. format(retries))) return True except: logger.exception( 'Notification handler for external authz plugin caught exception and could not complete: {} {}' .format(notification_type, notification_type)) raise
def process_analyzer_job(system_user_auth, qobj, layer_cache_enable): global servicename #current_avg, current_avg_count timer = int(time.time()) event = None try: logger.debug('dequeued object: {}'.format(qobj)) record = qobj['data'] userId = record['userId'] imageDigest = record['imageDigest'] manifest = record['manifest'] user_record = catalog.get_user(system_user_auth, userId) user_auth = (user_record['userId'], user_record['password']) # check to make sure image is still in DB try: image_records = catalog.get_image(user_auth, imageDigest=imageDigest) if image_records: image_record = image_records[0] else: raise Exception("empty image record from catalog") except Exception as err: logger.warn( "dequeued image cannot be fetched from catalog - skipping analysis (" + str(imageDigest) + ") - exception: " + str(err)) return (True) logger.info("image dequeued for analysis: " + str(userId) + " : " + str(imageDigest)) if image_record[ 'analysis_status'] != anchore_engine.subsys.taskstate.base_state( 'analyze'): logger.debug( "dequeued image is not in base state - skipping analysis") return (True) try: logger.spew("TIMING MARK0: " + str(int(time.time()) - timer)) last_analysis_status = image_record['analysis_status'] image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.working_state( 'analyze') rc = catalog.update_image(user_auth, imageDigest, image_record) # disable the webhook call for image state transistion to 'analyzing' #try: # for image_detail in image_record['image_detail']: # fulltag = image_detail['registry'] + "/" + image_detail['repo'] + ":" + image_detail['tag'] # npayload = { # 'last_eval': {'imageDigest': imageDigest, 'analysis_status': last_analysis_status}, # 'curr_eval': {'imageDigest': imageDigest, 'analysis_status': image_record['analysis_status']}, # } # rc = anchore_engine.subsys.notifications.queue_notification(userId, fulltag, 'analysis_update', npayload) #except Exception as err: # logger.warn("failed to enqueue notification on image analysis state update - exception: " + str(err)) # actually do analysis registry_creds = catalog.get_registry(user_auth) try: image_data = perform_analyze( userId, manifest, image_record, registry_creds, layer_cache_enable=layer_cache_enable) except AnchoreException as e: event = events.AnalyzeImageFail(user_id=userId, image_digest=imageDigest, error=e.to_dict()) raise imageId = None try: imageId = image_data[0]['image']['imageId'] except Exception as err: logger.warn( "could not get imageId after analysis or from image record - exception: " + str(err)) try: logger.debug("archiving analysis data") rc = catalog.put_document(user_auth, 'analysis_data', imageDigest, image_data) except Exception as e: err = CatalogClientError( msg='Failed to upload analysis data to catalog', cause=e) event = events.ArchiveAnalysisFail(user_id=userId, image_digest=imageDigest, error=err.to_dict()) raise err if rc: try: logger.debug("extracting image content data") image_content_data = {} for content_type in anchore_engine.services.common.image_content_types + anchore_engine.services.common.image_metadata_types: try: image_content_data[ content_type] = anchore_engine.services.common.extract_analyzer_content( image_data, content_type, manifest=manifest) except: image_content_data[content_type] = {} if image_content_data: logger.debug("adding image content data to archive") rc = catalog.put_document(user_auth, 'image_content_data', imageDigest, image_content_data) try: logger.debug( "adding image analysis data to image_record") anchore_engine.services.common.update_image_record_with_analysis_data( image_record, image_data) except Exception as err: raise err except Exception as err: logger.warn( "could not store image content metadata to archive - exception: " + str(err)) logger.debug("adding image record to policy-engine service (" + str(userId) + " : " + str(imageId) + ")") try: if not imageId: raise Exception( "cannot add image to policy engine without an imageId" ) localconfig = anchore_engine.configuration.localconfig.get_config( ) verify = localconfig['internal_ssl_verify'] client = anchore_engine.clients.policy_engine.get_client( user=system_user_auth[0], password=system_user_auth[1], verify_ssl=verify) try: logger.debug( "clearing any existing record in policy engine for image: " + str(imageId)) rc = client.delete_image(user_id=userId, image_id=imageId) except Exception as err: logger.warn("exception on pre-delete - exception: " + str(err)) logger.info('Loading image: {} {}'.format(userId, imageId)) request = ImageIngressRequest( user_id=userId, image_id=imageId, fetch_url='catalog://' + str(userId) + '/analysis_data/' + str(imageDigest)) logger.debug("policy engine request: " + str(request)) resp = client.ingress_image(request) logger.debug("policy engine image add response: " + str(resp)) except Exception as err: import traceback traceback.print_exc() newerr = PolicyEngineClientError( msg='Adding image to policy-engine failed', cause=str(err)) event = events.LoadAnalysisFail(user_id=userId, image_digest=imageDigest, error=newerr.to_dict()) raise newerr logger.debug("updating image catalog record analysis_status") last_analysis_status = image_record['analysis_status'] image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.complete_state( 'analyze') image_record['analyzed_at'] = int(time.time()) rc = catalog.update_image(user_auth, imageDigest, image_record) try: annotations = {} try: if image_record.get('annotations', '{}'): annotations = json.loads( image_record.get('annotations', '{}')) except Exception as err: logger.warn( "could not marshal annotations from json - exception: " + str(err)) for image_detail in image_record['image_detail']: fulltag = image_detail['registry'] + "/" + image_detail[ 'repo'] + ":" + image_detail['tag'] last_payload = { 'imageDigest': imageDigest, 'analysis_status': last_analysis_status, 'annotations': annotations } curr_payload = { 'imageDigest': imageDigest, 'analysis_status': image_record['analysis_status'], 'annotations': annotations } npayload = { 'last_eval': last_payload, 'curr_eval': curr_payload, } if annotations: npayload['annotations'] = annotations rc = anchore_engine.subsys.notifications.queue_notification( userId, fulltag, 'analysis_update', npayload) except Exception as err: logger.warn( "failed to enqueue notification on image analysis state update - exception: " + str(err)) else: err = CatalogClientError( msg='Failed to upload analysis data to catalog', cause='Invalid response from catalog API - {}'.format( str(rc))) event = events.ArchiveAnalysisFail(user_id=userId, image_digest=imageDigest, error=err.to_dict()) raise err logger.info("analysis complete: " + str(userId) + " : " + str(imageDigest)) logger.spew("TIMING MARK1: " + str(int(time.time()) - timer)) try: run_time = float(time.time() - timer) #current_avg_count = current_avg_count + 1.0 #new_avg = current_avg + ((run_time - current_avg) / current_avg_count) #current_avg = new_avg anchore_engine.subsys.metrics.histogram_observe( 'anchore_analysis_time_seconds', run_time, buckets=[ 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0, 3600.0 ], status="success") #anchore_engine.subsys.metrics.counter_inc('anchore_images_analyzed_total') #localconfig = anchore_engine.configuration.localconfig.get_config() #service_record = {'hostid': localconfig['host_id'], 'servicename': servicename} #anchore_engine.subsys.servicestatus.set_status(service_record, up=True, available=True, detail={'avg_analysis_time_sec': current_avg, 'total_analysis_count': current_avg_count}, update_db=True) except Exception as err: logger.warn(str(err)) pass except Exception as err: run_time = float(time.time() - timer) logger.exception("problem analyzing image - exception: " + str(err)) anchore_engine.subsys.metrics.histogram_observe( 'anchore_analysis_time_seconds', run_time, buckets=[ 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0, 3600.0 ], status="fail") image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.fault_state( 'analyze') image_record[ 'image_status'] = anchore_engine.subsys.taskstate.fault_state( 'image_status') rc = catalog.update_image(user_auth, imageDigest, image_record) finally: if event: try: catalog.add_event(user_auth, event) except: logger.error( 'Ignoring error creating analysis failure event') except Exception as err: logger.warn("job processing bailed - exception: " + str(err)) raise err return (True)
def perform_analyze_localanchore(userId, manifest, image_record, registry_creds, layer_cache_enable=False): ret_analyze = {} localconfig = anchore_engine.configuration.localconfig.get_config() do_docker_cleanup = localconfig['cleanup_images'] try: image_detail = image_record['image_detail'][0] registry_manifest = manifest pullstring = image_detail['registry'] + "/" + image_detail[ 'repo'] + "@" + image_detail['imageDigest'] fulltag = image_detail['registry'] + "/" + image_detail[ 'repo'] + ":" + image_detail['tag'] logger.debug("using pullstring (" + str(pullstring) + ") and fulltag (" + str(fulltag) + ") to pull image data") except Exception as err: image_detail = pullstring = fulltag = None raise Exception( "failed to extract requisite information from image_record - exception: " + str(err)) timer = int(time.time()) logger.spew("TIMING MARK0: " + str(int(time.time()) - timer)) logger.debug("obtaining anchorelock..." + str(pullstring)) with localanchore.get_anchorelock(lockId=pullstring): logger.debug("obtaining anchorelock successful: " + str(pullstring)) logger.spew("TIMING MARK1: " + str(int(time.time()) - timer)) logger.info("performing analysis on image: " + str(pullstring)) # pull the digest, but also any tags associated with the image (that we know of) in order to populate the local docker image try: rc = localanchore.pull(userId, pullstring, image_detail, pulltags=True, registry_creds=registry_creds) if not rc: raise Exception("anchore analyze failed:") pullstring = re.sub("sha256:", "", rc['Id']) image_detail['imageId'] = pullstring except Exception as err: logger.error("error on pull: " + str(err)) raise err logger.spew("TIMING MARK2: " + str(int(time.time()) - timer)) # analyze! try: rc = localanchore.analyze(pullstring, image_detail) if not rc: raise Exception("anchore analyze failed:") except Exception as err: logger.error("error on analyze: " + str(err)) raise err logger.spew("TIMING MARK3: " + str(int(time.time()) - timer)) # get the result from anchore logger.debug("retrieving image data from anchore") try: image_data = localanchore.get_image_export(pullstring, image_detail) if not image_data: raise Exception("anchore image data export failed:") except Exception as err: logger.error("error on image export: " + str(err)) raise err logger.spew("TIMING MARK5: " + str(int(time.time()) - timer)) try: logger.debug("removing image: " + str(pullstring)) rc = localanchore.remove_image(pullstring, docker_remove=do_docker_cleanup, anchore_remove=True) logger.debug("removing image complete: " + str(pullstring)) except Exception as err: raise err logger.spew("TIMING MARK6: " + str(int(time.time()) - timer)) ret_analyze = image_data logger.info("performing analysis on image complete: " + str(pullstring)) return (ret_analyze)
def get_image_manifest_skopeo(url, registry, repo, intag=None, indigest=None, user=None, pw=None, verify=True): manifest = {} digest = None testDigest = None if indigest: pullstring = registry + "/" + repo + "@" + indigest elif intag: pullstring = registry + "/" + repo + ":" + intag else: raise Exception( "invalid input - must supply either an intag or indigest") try: if user and pw: os.environ['SKOPUSER'] = user os.environ['SKOPPASS'] = pw credstr = "--creds ${SKOPUSER}:${SKOPPASS}" credstr = "--creds " + user + ":" + pw else: credstr = "" if verify: tlsverifystr = "--tls-verify=true" else: tlsverifystr = "--tls-verify=false" try: cmdstr = "skopeo inspect --raw " + tlsverifystr + " " + credstr + " docker://" + pullstring try: rc, sout, serr = anchore_engine.services.common.run_command( cmdstr) if rc != 0: raise Exception("command failed: cmd=" + str(cmdstr) + " exitcode=" + str(rc) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) #testDigest = anchore_engine.services.common.manifest_to_digest(sout) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err digest = anchore_engine.services.common.manifest_to_digest(sout) manifest = json.loads(sout) except Exception as err: logger.warn("CMD failed - exception: " + str(err)) digest = None manifest = {} #try: # cmdstr = "skopeo inspect "+tlsverifystr+" "+credstr+" docker://"+pullstring # try: # rc, sout, serr = anchore_engine.services.common.run_command(cmdstr) # if rc != 0: # raise Exception("command failed: cmd="+str(cmdstr)+" exitcode="+str(rc)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip()) # else: # logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip()) # except Exception as err: # logger.error("command failed with exception - " + str(err)) # raise err # skopeo_output = sout # data = json.loads(skopeo_output) # digest = data['Digest'] #except Exception as err: # logger.warn("CMD failed - exception: " + str(err)) # digest = None except Exception as err: #logger.error("error in skopeo wrapper - exception: " + str(err)) raise err finally: try: del os.environ['SKOPUSER'] except: pass try: del os.environ['SKOPPASS'] except: pass if not manifest or not digest: raise Exception("no digest/manifest from skopeo") #logger.debug("digest test comparison: " + str(digest == testDigest) + " : " + str(digest) + " : " + str(testDigest)) return (manifest, digest)
def default_monitor_func(**kwargs): """ Generic monitor thread function for invoking tasks defined in a monitor dict :param kwargs: :return: """ global click, running, last_run my_monitors = kwargs['monitors'] monitor_threads = kwargs['monitor_threads'] servicename = kwargs['servicename'] timer = int(time.time()) if click < 5: click = click + 1 logger.debug("service ("+str(servicename)+") starting in: " + str(5 - click)) return True if round(time.time() - last_run) < kwargs['kick_timer']: logger.spew( "timer hasn't kicked yet: " + str(round(time.time() - last_run)) + " : " + str(kwargs['kick_timer'])) return True try: running = True last_run = time.time() # handle setting the cycle timers based on configuration for monitor_name in list(my_monitors.keys()): if not my_monitors[monitor_name]['initialized']: # first time if 'cycle_timers' in kwargs and monitor_name in kwargs['cycle_timers']: try: the_cycle_timer = my_monitors[monitor_name]['cycle_timer'] min_cycle_timer = my_monitors[monitor_name]['min_cycle_timer'] max_cycle_timer = my_monitors[monitor_name]['max_cycle_timer'] config_cycle_timer = int(kwargs['cycle_timers'][monitor_name]) if config_cycle_timer < 0: the_cycle_timer = abs(int(config_cycle_timer)) elif config_cycle_timer == 0: my_monitors[monitor_name]['enabled'] = False logger.debug("monitor '{}' has been explicitly disabled in config".format(monitor_name)) elif config_cycle_timer < min_cycle_timer: logger.warn("configured cycle timer for handler ("+str(monitor_name)+") is less than the allowed min ("+str(min_cycle_timer)+") - using allowed min") the_cycle_timer = min_cycle_timer elif config_cycle_timer > max_cycle_timer: logger.warn("configured cycle timer for handler ("+str(monitor_name)+") is greater than the allowed max ("+str(max_cycle_timer)+") - using allowed max") the_cycle_timer = max_cycle_timer else: the_cycle_timer = config_cycle_timer my_monitors[monitor_name]['cycle_timer'] = the_cycle_timer except Exception as err: logger.warn("exception setting custom cycle timer for handler ("+str(monitor_name)+") - using default") my_monitors[monitor_name]['initialized'] = True # handle the thread (re)starters here for monitor_name in list(my_monitors.keys()): if my_monitors[monitor_name].get('enabled', True): start_thread = False if monitor_name not in monitor_threads: start_thread = True else: if not monitor_threads[monitor_name].isAlive(): logger.debug("thread stopped - restarting: " + str(monitor_name)) monitor_threads[monitor_name].join() start_thread = True if start_thread: monitor_threads[monitor_name] = threading.Thread(target=my_monitors[monitor_name]['handler'], args=my_monitors[monitor_name]['args'], kwargs={'mythread': my_monitors[monitor_name]}) logger.debug("starting up monitor_thread: " + str(monitor_name)) monitor_threads[monitor_name].start() except Exception as err: logger.error(str(err)) finally: running = False return True
def get_image_manifest_dockerhub_orig(repo, tag, user=None, pw=None): manifest = {} digest = "" try: if not user or not pw: authy = None else: authy = (user, pw) #TODO externalize URLs auth_url = "https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repository}:pull" url = auth_url.format(repository=repo) token = "" try: r = requests.get(url, json=True, auth=authy) if r.status_code == 200: #token = requests.get(url, json=True, auth=authy).json()["token"] token = r.json()["token"] elif r.status_code == 402: raise Exception( "not authorized (401) returned from registry: auth_url=(" + str(url) + ") user=(" + str(user) + ")") else: raise Exception("got bad code (" + str(r.status_code) + ") from manifest request: " + str(r.text)) except Exception as err: logger.error("could not get auth token: " + str(err)) raise err get_manifest_template = "https://registry.hub.docker.com/v2/{repository}/manifests/{tag}" url = get_manifest_template.format(repository=repo, tag=tag) try: headers = { "Authorization": "Bearer {}".format(token), "Accept": "application/vnd.docker.distribution.manifest.v2+json" } r = requests.get(url, headers=headers, json=True) if r.status_code == 200: manifest = r.json() digest = r.headers['Docker-Content-Digest'] elif r.status_code == 401: raise Exception( "not authorized (401) returned from registry: registry=(https://registry.hub.docker.com) repo=(" + str(repo) + ") tag=(" + str(tag) + ") user=(" + str(user) + ")") else: raise Exception("got bad code (" + str(r.status_code) + ") from manifest request: " + str(r.text)) except Exception as err: logger.warn("could not get manifest: " + str(err)) raise err except Exception as err: raise err return (manifest, digest)
def get_image_manifest_skopeo(url, registry, repo, intag=None, indigest=None, user=None, pw=None, verify=True): manifest = {} digest = None testDigest = None if indigest: pullstring = registry + "/" + repo + "@" + indigest elif intag: pullstring = registry + "/" + repo + ":" + intag else: raise Exception( "invalid input - must supply either an intag or indigest") try: proc_env = os.environ.copy() if user and pw: proc_env['SKOPUSER'] = user proc_env['SKOPPASS'] = pw credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"' else: credstr = "" if verify: tlsverifystr = "--tls-verify=true" else: tlsverifystr = "--tls-verify=false" try: cmd = [ "/bin/sh", "-c", "skopeo inspect --raw {} {} docker://{}".format( tlsverifystr, credstr, pullstring) ] cmdstr = ' '.join(cmd) try: rc, sout, serr = anchore_engine.services.common.run_command_list( cmd, env=proc_env) if rc != 0: raise Exception("command failed: cmd=" + str(cmdstr) + " exitcode=" + str(rc) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err digest = anchore_engine.services.common.manifest_to_digest(sout) manifest = json.loads(sout) except Exception as err: logger.warn("CMD failed - exception: " + str(err)) digest = None manifest = {} except Exception as err: raise err if not manifest or not digest: raise Exception("no digest/manifest from skopeo") return (manifest, digest)
def makeService(snames, options, db_connect=True, require_system_user_auth=True, module_name="anchore_engine.services", validate_params={}): try: logger.enable_bootstrap_logging(service_name=','.join(snames)) try: # config and init configfile = configdir = None if options['config']: configdir = options['config'] configfile = os.path.join(options['config'], 'config.yaml') anchore_engine.configuration.localconfig.load_config( configdir=configdir, configfile=configfile, validate_params=validate_params) localconfig = anchore_engine.configuration.localconfig.get_config() localconfig['myservices'] = [] logger.spew("localconfig=" + json.dumps(localconfig, indent=4, sort_keys=True)) except Exception as err: logger.error("cannot load configuration: exception - " + str(err)) raise err # get versions of things try: versions = anchore_engine.configuration.localconfig.get_versions() except Exception as err: logger.error("cannot detect versions of service: exception - " + str(err)) raise err if db_connect: logger.info("initializing database") # connect to DB try: db.initialize(localconfig=localconfig, versions=versions) except Exception as err: logger.error("cannot connect to configured DB: exception - " + str(err)) raise err #credential bootstrap localconfig['system_user_auth'] = (None, None) if require_system_user_auth: gotauth = False max_retries = 60 for count in range(1, max_retries): if gotauth: continue try: with session_scope() as dbsession: localconfig[ 'system_user_auth'] = get_system_user_auth( session=dbsession) if localconfig['system_user_auth'] != (None, None): gotauth = True else: logger.error( "cannot get system user auth credentials yet, retrying (" + str(count) + " / " + str(max_retries) + ")") time.sleep(5) except Exception as err: logger.error( "cannot get system-user auth credentials - service may not have system level access" ) localconfig['system_user_auth'] = (None, None) if not gotauth: raise Exception( "service requires system user auth to start") # application object application = service.Application("multi-service-" + '-'.join(snames)) #multi-service retservice = service.MultiService() retservice.setServiceParent(application) success = False try: scount = 0 for sname in snames: if sname in localconfig['services'] and localconfig[ 'services'][sname]['enabled']: smodule = importlib.import_module(module_name + "." + sname) s = smodule.createService(sname, localconfig) s.setServiceParent(retservice) rc = smodule.initializeService(sname, localconfig) if not rc: raise Exception("failed to initialize service") rc = smodule.registerService(sname, localconfig) if not rc: raise Exception("failed to register service") logger.debug("starting service: " + sname) success = True scount += 1 localconfig['myservices'].append(sname) else: logger.error( "service not enabled in config, not starting service: " + sname) if scount == 0: logger.error( "no services/subservices were enabled/started on this host" ) success = False except Exception as err: logger.error("cannot create/init/register service: " + sname + " - exception: " + str(err)) success = False if not success: logger.error("cannot start service (see above for information)") traceback.print_exc('Service init failure') raise Exception("cannot start service (see above for information)") return (retservice) finally: logger.disable_bootstrap_logging()
def get_authenticated_cli(userId, registry, registry_creds=[]): global docker_cli_unauth, docker_clis logger.debug( "DOCKER CLI: entering auth cli create/fetch for input user/registry: " + str(userId) + " / " + str(registry)) localconfig = anchore_engine.configuration.localconfig.get_config() if not userId: if not docker_cli_unauth: docker_cli_unauth = docker.Client( base_url=localconfig['docker_conn'], version='auto', timeout=int(localconfig['docker_conn_timeout'])) logger.debug("DOCKER CLI: returning unauth client") return (docker_cli_unauth) if userId in docker_clis and registry in docker_clis[userId]: if 'registry_creds' in docker_clis[userId][ registry] and registry_creds == docker_clis[userId][registry][ 'registry_creds']: logger.debug("DOCKER CLI: found existing authenticated CLI") return (docker_clis[userId][registry]['cli']) else: logger.debug("DOCKER CLI: detected cred change, will refresh CLI") logger.debug("DOCKER CLI: making new auth CLI for user/registry: " + str(userId) + " / " + str(registry)) try: if userId not in docker_clis: docker_clis[userId] = {} if registry not in docker_clis[userId]: docker_clis[userId][registry] = {} user = pw = None for registry_record in registry_creds: if registry_record['registry'] == registry: user, pw = anchore_engine.auth.common.get_docker_registry_userpw( registry_record) if not user or not pw: logger.debug("DOCKER CLI: making unauth CLI") docker_clis[userId][registry]['cli'] = docker.Client( base_url=localconfig['docker_conn'], version='auto', timeout=int(localconfig['docker_conn_timeout'])) docker_clis[userId][registry]['registry_creds'] = [] else: logger.debug("DOCKER CLI: making auth CLI") try: cli = docker.Client(base_url=localconfig['docker_conn'], version='auto', timeout=int( localconfig['docker_conn_timeout'])) rc = cli.login(user, password=pw, registry=registry, reauth=False) docker_clis[userId][registry]['cli'] = cli docker_clis[userId][registry][ 'registry_creds'] = registry_creds except Exception as err: logger.error("DOCKER CLI auth err: " + str(err)) raise err except Exception as err: logger.error("DOCKER CLI: unable to get docker cli - exception: " + str(err)) raise err if userId in docker_clis and registry in docker_clis[userId]: logger.debug("DOCKER CLI: returning auth client") return (docker_clis[userId][registry]['cli']) logger.error( "DOCKER CLI: unable to complete authenticated client create/fetch") raise Exception( "DOCKER CLI: unable to complete authenticated client create/fetch") return (None)
def sync_metadata( source_feeds: Dict[ str, Dict[str, Union[FeedAPIRecord, List[FeedAPIGroupRecord]]] ], to_sync: List[str] = None, operation_id: Optional[str] = None, groups: bool = True, ) -> Tuple[Dict[str, FeedMetadata], List[Tuple[str, Union[str, BaseException]]]]: """ Get metadata from source and sync db metadata records to that (e.g. add any new groups or feeds) Executes as a unit-of-work for db, so will commit result and returns the records found on upstream source. If a record exists in db but was not found upstream, it is not returned :param source_feeds: mapping containing FeedAPIRecord and FeedAPIGroupRecord :type source_feeds: Dict[str, Dict[str, Union[FeedAPIRecord, List[FeedAPIGroupRecord]]]] :param to_sync: list of string feed names to sync metadata on :type to_sync: List[str] :param operation_id: UUID4 hexadecimal string :type operation_id: Optional[str] :param groups: whether or not to sync group metadata (defaults to True, which will sync group metadata) :type groups: bool :return: tuple, first element: dict of names mapped to db records post-sync only including records successfully updated by upstream, second element is a list of tuples where each tuple is (failed_feed_name, error_obj) :rtype: Tuple[Dict[str, FeedMetadata], List[Tuple[str, Union[str, BaseException]]] """ if not to_sync: return {}, [] db = get_session() try: logger.info( "Syncing feed and group metadata from upstream source (operation_id={})".format( operation_id ) ) failed = [] db_feeds = MetadataSyncUtils._pivot_and_filter_feeds_by_config( to_sync, list(source_feeds.keys()), get_all_feeds(db) ) for feed_name, feed_api_record in source_feeds.items(): try: logger.info( "Syncing metadata for feed: {} (operation_id={})".format( feed_name, operation_id ) ) feed_metadata_map = MetadataSyncUtils._sync_feed_metadata( db, feed_api_record, db_feeds, operation_id ) if groups: MetadataSyncUtils._sync_feed_group_metadata( db, feed_api_record, feed_metadata_map, operation_id ) except Exception as e: logger.exception("Error syncing feed {}".format(feed_name)) logger.warn( "Could not sync metadata for feed: {} (operation_id={})".format( feed_name, operation_id ) ) failed.append((feed_name, e)) finally: db.flush() # Reload db_feeds = MetadataSyncUtils._pivot_and_filter_feeds_by_config( to_sync, list(source_feeds.keys()), get_all_feeds(db) ) db.commit() logger.info( "Metadata sync from feeds upstream source complete (operation_id={})".format( operation_id ) ) return db_feeds, failed except Exception as e: logger.error( "Rolling back feed metadata update due to error: {} (operation_id={})".format( e, operation_id ) ) db.rollback() raise
def call_api( self, method: callable, path: str, path_params=None, query_params=None, extra_headers=None, body=None, ): """ Invoke the api :param method: requests function to invoke (eg. requests.get) :param path: url path :param path_params: path params as dict :param query_params: query param dict :param extra_headers: header map to merge with default headers for this client :param body: string body to send :return: """ if path_params: path_params = { name: urlparse.quote(value) for name, value in path_params.items() } final_url = urlparse.urljoin(self.url, path.format(**path_params)) else: final_url = urlparse.urljoin(self.url, path) request_headers = copy.copy(self.__headers__) if extra_headers: request_headers.update(extra_headers) # Remove any None valued query params if query_params: filtered_qry_params = { k: v for k, v in filter(lambda x: x[1] is not None, query_params.items()) } else: filtered_qry_params = None logger.debug( "Dispatching: url={url}, headers={headers}, body={body}, params={params}" .format( url=final_url, headers=request_headers, body=body[:512] + ("..." if len(body) > 512 else "") if body else body, params=filtered_qry_params, )) try: if self.username and self.password: return method( url=final_url, headers=request_headers, data=body, params=filtered_qry_params, auth=(self.username, self.password), ) else: return method( url=final_url, headers=request_headers, data=body, params=filtered_qry_params, ) except Exception as e: logger.error("Failed client call to url: {}. Response: {}".format( final_url, e.__dict__)) raise e
def init_oauth(app, grant_types, expiration_config): """ Configure the oauth routes and handlers via authlib :return: """ logger.debug("Initializing oauth routes") try: tok_mgr = token_manager() logger.info("Initialized the token manager") except OauthNotConfiguredError: logger.info("OAuth support not configured, cannot initialize it") return None except InvalidOauthConfigurationError: logger.error("OAuth has invalid configuration, cannot initialize it") raise def query_client(client_id): db = get_session() c = db.query(OAuth2Client).filter_by(client_id=client_id).first() return c def do_not_save_token(token, request): return None # Don't use this (yet), due to token signing that allows system to verify without persistence def save_token(token, request): try: if request.user: user_id = request.user.username else: user_id = None client = request.client tok = OAuth2Token(client_id=client.client_id, user_id=user_id, **token) db = get_session() db.add(tok) db.commit() except: logger.exception("Exception saving token") raise try: # Initialize an anonymous client record with session_scope() as db: f = db.query(OAuth2Client).filter_by(client_id="anonymous").first() if not f: c = OAuth2Client() c.client_id = "anonymous" c.user_id = None c.client_secret = None c.issued_at = time.time() - 100 c.expires_at = time.time() + 1000 c.grant_type = "password" c.token_endpoint_auth_method = "none" c.client_name = "anonymous" db.add(c) except Exception as e: logger.debug("Default client record init failed: {}".format(e)) app.config["OAUTH2_JWT_ENABLED"] = True app.config["OAUTH2_ACCESS_TOKEN_GENERATOR"] = generate_token app.config["OAUTH2_REFRESH_TOKEN_GENERATOR"] = False # Only the password grant type is used, others can stay defaults app.config["OAUTH2_TOKEN_EXPIRES_IN"] = expiration_config app.config["OAUTH2_JWT_KEY"] = tok_mgr.default_issuer().signing_key app.config["OAUTH2_JWT_ISS"] = tok_mgr.default_issuer().issuer app.config["OAUTH2_JWT_ALG"] = tok_mgr.default_issuer().signing_alg authz = AuthorizationServer(app, query_client=query_client, save_token=do_not_save_token) # Support only the password grant for now for grant in grant_types: logger.debug("Registering oauth grant handler: {}".format( getattr(grant, "GRANT_TYPE", "unknown"))) authz.register_grant(grant) logger.debug("Oauth init complete") return authz
def squash(unpackdir, cachedir, layers): rootfsdir = unpackdir + "/rootfs" if os.path.exists(unpackdir + "/squashed.tar"): return (True) whpatt = re.compile("\.wh\..*") whopqpatt = re.compile("\.wh\.\.wh\.\.opq") #slashprefixpatt = re.compile("^[\./|/]+") slashprefixpatt = re.compile("^/+|\.+/+") tarfiles = {} tarfiles_members = {} fhistory = {} try: logger.debug("Layers to process: {}".format(layers)) logger.debug("Pass 1: generating layer file timeline") deferred_hardlinks_destination = {} hardlink_destinations = {} for l in layers: htype, layer = l.split(":", 1) layertar = get_layertarfile(unpackdir, cachedir, layer) ltf = None try: lfhistory = {} deferred_hardlinks = {} logger.debug("processing layer {} - {}".format(l, layertar)) tarfiles[l] = tarfile.open(layertar, mode='r', format=tarfile.PAX_FORMAT) tarfiles_members[l] = {} for member in tarfiles[l].getmembers(): # clean up any prefix on the member names for history tracking purposes tarfilename = member.name member.name = slashprefixpatt.sub("", member.name) if member.islnk() and member.linkname: member.linkname = slashprefixpatt.sub( "", member.linkname) member.linkpath = member.linkname member.pax_headers['path'] = member.name # regular processing starts here tarfiles_members[l][member.name] = member filename = member.name if filename not in lfhistory: lfhistory[filename] = {} lfhistory[filename]['latest_layer_tar'] = l lfhistory[filename]['exists'] = True if whopqpatt.match(os.path.basename(filename)): # never include the wh itself lfhistory[filename]['exists'] = False # found an opq entry, which means that this files in the next layer down (only) should not be included fsub = re.sub(r"\.wh\.\.wh\.\.opq", "", filename, 1) fsub = re.sub("/+$", "", fsub) for other_filename in fhistory.keys(): if re.match("^{}/".format(re.escape(fsub)), other_filename): if other_filename not in lfhistory: lfhistory[other_filename] = {} lfhistory[other_filename].update( fhistory[other_filename]) lfhistory[other_filename]['exists'] = False elif whpatt.match(os.path.basename(filename)): # never include the wh itself lfhistory[filename]['exists'] = False fsub = re.sub(r"\.wh\.", "", filename, 1) if fsub not in lfhistory: lfhistory[fsub] = {} if fsub in fhistory: lfhistory[fsub].update(fhistory[fsub]) lfhistory[fsub]['exists'] = False for other_filename in fhistory.keys(): if re.match("^{}/".format(re.escape(fsub)), other_filename): if other_filename not in lfhistory: lfhistory[other_filename] = {} lfhistory[other_filename].update( fhistory[other_filename]) lfhistory[other_filename]['exists'] = False if lfhistory[filename]['exists'] and member.islnk(): el = { 'hl_target_layer': l, 'hl_target_name': member.linkname, 'hl_replace': False, } lfhistory[filename].update(el) if member.linkname not in hardlink_destinations: hardlink_destinations[member.linkname] = [] el = { 'filename': filename, 'layer': l, } hardlink_destinations[member.linkname].append(el) for filename in list(lfhistory.keys()): if filename in hardlink_destinations: for el in hardlink_destinations[filename]: if el['layer'] != l: if el['filename'] not in lfhistory: lfhistory[el['filename']] = {} lfhistory[el['filename']].update( fhistory[el['filename']]) lfhistory[el['filename']]['hl_replace'] = True fhistory.update(lfhistory) except Exception as err: logger.error( "layer handler failure - exception: {}".format(err)) raise (err) logger.debug("Pass 2: creating squashtar from layers") allexcludes = [] with tarfile.open(os.path.join(unpackdir, "squashed.tar"), mode='w', format=tarfile.PAX_FORMAT) as oltf: imageSize = 0 deferred_hardlinks = {} for l in tarfiles_members.keys(): for filename in tarfiles_members[l].keys(): if fhistory[filename]['exists'] and fhistory[filename][ 'latest_layer_tar'] == l: member = tarfiles_members[l].get(filename) if member.isreg(): memberfd = tarfiles[l].extractfile(member) oltf.addfile(member, fileobj=memberfd) elif member.islnk(): if fhistory[filename]['hl_replace']: deferred_hardlinks[filename] = fhistory[ filename] else: oltf.addfile(member) else: oltf.addfile(member) for filename in deferred_hardlinks.keys(): l = fhistory[filename]['latest_layer_tar'] member = tarfiles_members[l].get(filename) logger.debug("deferred hardlink {}".format(fhistory[filename])) try: logger.debug( "attempt to lookup deferred {} content source".format( filename)) content_layer = fhistory[filename]['hl_target_layer'] content_filename = fhistory[filename]['hl_target_name'] logger.debug( "attempt to extract deferred {} from layer {} (for lnk {})" .format(content_filename, content_layer, filename)) content_member = tarfiles_members[content_layer].get( content_filename) content_memberfd = tarfiles[content_layer].extractfile( content_member) logger.debug( "attempt to construct new member for deferred {}". format(filename)) new_member = copy.deepcopy(content_member) new_member.name = member.name new_member.pax_headers['path'] = member.name logger.debug( "attempt to add final to squashed tar {} -> {}".format( filename, new_member.name)) oltf.addfile(new_member, fileobj=content_memberfd) except Exception as err: import traceback traceback.print_exc() logger.warn( "failed to store hardlink ({} -> {}) - exception: {}". format(member.name, member.linkname, err)) finally: logger.debug("Pass 3: closing layer tarfiles") for l in tarfiles.keys(): if tarfiles[l]: try: tarfiles[l].close() except Exception as err: logger.error( "failure closing tarfile {} - exception: {}".format( l, err)) imageSize = 0 if os.path.exists(os.path.join(unpackdir, "squashed.tar")): imageSize = os.path.getsize(os.path.join(unpackdir, "squashed.tar")) return ("done", imageSize)
def upgrade(anchore_module, dontask, skip_db_compat_check): """ Run a Database Upgrade idempotently. If database is not initialized yet, but can be connected, then exit cleanly with status = 0, if no connection available then return error. Otherwise, upgrade from the db running version to the code version and exit. """ ecode = 0 if not anchore_module: module_name = "anchore_engine" else: module_name = str(anchore_module) try: try: logger.info("Loading DB upgrade routines from module.") module = importlib.import_module(module_name + ".db.entities.upgrade") except Exception as err: raise Exception("Input anchore-module (" + str(module_name) + ") cannot be found/imported - exception: " + str(err)) code_versions, db_versions = anchore_manager.cli.utils.init_database(upgrade_module=module, do_db_compatibility_check=(not skip_db_compat_check)) code_db_version = code_versions.get('db_version', None) running_db_version = db_versions.get('db_version', None) if not code_db_version or not running_db_version: raise Exception("cannot get version information (code_db_version={} running_db_version={})".format(code_db_version, running_db_version)) elif code_db_version == running_db_version: logger.info("Code and DB versions are in sync.") ecode = 0 else: logger.info("Detected anchore-engine version {}, running DB version {}.".format(code_db_version, running_db_version)) do_upgrade = False if dontask: do_upgrade = True else: try: answer = raw_input("Performing this operation requires *all* anchore-engine services to be stopped - proceed? (y/N)") except: answer = "n" if 'y' == answer.lower(): do_upgrade = True if do_upgrade: logger.info("Performing upgrade.") try: # perform the upgrade logic here rc = module.run_upgrade() if rc: logger.info("Upgrade completed") else: logger.info("No upgrade necessary. Completed.") except Exception as err: raise err else: logger.info("Skipping upgrade.") except Exception as err: logger.error(anchore_manager.cli.utils.format_error_output(config, 'dbupgrade', {}, err)) if not ecode: ecode = 2 anchore_manager.cli.utils.doexit(ecode)
def run_anchore_analyzers(staging_dirs, imageDigest, imageId, localconfig): outputdir = staging_dirs['outputdir'] unpackdir = staging_dirs['unpackdir'] copydir = staging_dirs['copydir'] configdir = localconfig['service_dir'] # run analyzers #anchore_module_root = resource_filename("anchore", "anchore-modules") anchore_module_root = resource_filename("anchore_engine", "analyzers") analyzer_root = os.path.join(anchore_module_root, "modules") for f in list_analyzers(): #for f in os.listdir(analyzer_root): # thecmd = os.path.join(analyzer_root, f) # if re.match(".*\.py$", thecmd): cmdstr = " ".join( [f, configdir, imageId, unpackdir, outputdir, unpackdir]) if True: try: rc, sout, serr = utils.run_command(cmdstr) sout = utils.ensure_str(sout) serr = utils.ensure_str(serr) if rc != 0: raise Exception("command failed: cmd=" + str(cmdstr) + " exitcode=" + str(rc) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) #raise err analyzer_report = {} for analyzer_output in os.listdir( os.path.join(outputdir, "analyzer_output")): if analyzer_output not in analyzer_report: analyzer_report[analyzer_output] = {} for analyzer_output_el in os.listdir( os.path.join(outputdir, "analyzer_output", analyzer_output)): if analyzer_output_el not in analyzer_report[analyzer_output]: analyzer_report[analyzer_output][analyzer_output_el] = { 'base': {} } data = read_kvfile_todict( os.path.join(outputdir, "analyzer_output", analyzer_output, analyzer_output_el)) if data: analyzer_report[analyzer_output][analyzer_output_el][ 'base'] = read_kvfile_todict( os.path.join(outputdir, "analyzer_output", analyzer_output, analyzer_output_el)) else: analyzer_report[analyzer_output].pop(analyzer_output_el, None) if not analyzer_report[analyzer_output]: analyzer_report.pop(analyzer_output, None) return (analyzer_report)
def _get_system_user_credentials(self): """ Returns an AccessCredential object representing the system user :return: """ cred = None exp = None # Credential expiration, if needed logger.debug("Loading system user creds") with IdentityManager._cache_lock: cached_cred = IdentityManager._credential_cache.lookup( localconfig.SYSTEM_USERNAME) if cached_cred is not None: if cached_cred.is_expired(): # Flush it logger.debug( "Cached system credential is expired, flushing") IdentityManager._credential_cache.delete( localconfig.SYSTEM_USERNAME) else: logger.debug("Cached system credential still ok") # Use it cred = cached_cred if cred is None: logger.debug("Doing refresh/initial system cred load") try: tok_mgr = token_manager() except OauthNotConfiguredError: tok_mgr = None # Generate one if tok_mgr: # Generate a token usr = db_account_users.get(localconfig.SYSTEM_USERNAME, session=self.session) system_user_uuid = usr["uuid"] tok, exp = tok_mgr.generate_token(system_user_uuid, return_expiration=True) logger.debug( "Generated token with expiration {}".format(exp)) cred = HttpBearerCredential(tok, exp) else: rec = db_accounts.get(localconfig.SYSTEM_USERNAME, session=self.session) usr = db_account_users.get(localconfig.SYSTEM_USERNAME, session=self.session) if not rec or not usr: logger.error( "Could not find a system account or user. This is not an expected state" ) raise Exception("No system account or user found") # This will not work if the system admin has configured hashed passwords but not oauth. But, that should be caught at config validation. cred = HttpBasicCredential( usr["username"], usr.get("credentials", {}).get(UserAccessCredentialTypes.password, {}).get("value"), ) if cred is not None: logger.debug("Caching system creds") IdentityManager._credential_cache.cache_it( localconfig.SYSTEM_USERNAME, cred) return cred
def interactive_analyze(bodycontent): try: return_object = {} httpcode = 500 request_inputs = anchore_engine.services.common.do_request_prep( connexion.request, default_params={}) user_auth = request_inputs['auth'] method = request_inputs['method'] #bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] userId = request_inputs['userId'] try: #input prep #jsondata = json.loads(bodycontent) jsondata = bodycontent tag = jsondata.pop('tag', None) if not tag: httpcode = 500 raise Exception("must supply a valid tag param in json body") try: # image prep registry_creds = anchore_engine.clients.catalog.get_registry( user_auth) image_info = anchore_engine.services.common.get_image_info( userId, "docker", tag, registry_lookup=True, registry_creds=registry_creds) pullstring = image_info['registry'] + "/" + image_info[ 'repo'] + "@" + image_info['digest'] fulltag = image_info['registry'] + "/" + image_info[ 'repo'] + ":" + image_info['tag'] new_image_record = anchore_engine.services.common.make_image_record( userId, 'docker', fulltag, registry_lookup=False, registry_creds=(None, None)) image_detail = new_image_record['image_detail'][0] if not image_detail: raise Exception("no image found matching input") except Exception as err: httpcode = 404 raise Exception(str(err)) image_data, query_data = anchore_engine.services.analyzer.perform_analyze( userId, pullstring, fulltag, image_detail, registry_creds) if image_data: return_object = image_data httpcode = 200 else: httpcode = 500 raise Exception("analyze resulted in empty analysis data") except Exception as err: logger.error(str(err)) raise err except Exception as err: logger.error(str(err)) return_object = str(err) return (return_object, httpcode)
def _register(self): if not self.is_enabled: logger.error('Service not enabled in config, not registering service: ' + self.name) raise Exception('No service enabled, cannot continue bootstrap') logger.info('Registering service: {}'.format(self.name)) service_template = { 'type': 'anchore', 'base_url': 'N/A', 'status_base_url': 'N/A', 'version': 'v1', 'short_description': '' } hstring = 'http' if 'external_tls' in self.configuration: if self.configuration.get('external_tls', False): hstring = 'https' elif 'ssl_enable' in self.configuration: if self.configuration.get('ssl_enable', False): hstring = 'https' endpoint_hostname = endpoint_port = endpoint_hostport = None if self.configuration.get('external_hostname', False): endpoint_hostname = self.configuration.get('external_hostname') elif self.configuration.get('endpoint_hostname', False): endpoint_hostname = self.configuration.get('endpoint_hostname') if self.configuration.get('external_port', False): endpoint_port = int(self.configuration.get('external_port')) elif self.configuration.get('port', False): endpoint_port = int(self.configuration.get('port')) if endpoint_hostname: endpoint_hostport = endpoint_hostname if endpoint_port: endpoint_hostport = endpoint_hostport + ":" + str(endpoint_port) if endpoint_hostport: service_template['base_url'] = "{}://{}".format(hstring, endpoint_hostport) else: raise Exception("could not construct service base_url - please check service configuration for hostname/port settings") try: service_template['status'] = False service_template['status_message'] = taskstate.base_state('service_status') with session_scope() as dbsession: service_records = db_services.get_byname(self.__service_name__, session=dbsession) # fail if trying to add a service that must be unique in the system, but one already is registered in DB if self.__is_unique_service__: if len(service_records) > 1: raise Exception('more than one entry for service type (' + str( self.__service_name__) + ') exists in DB, but service must be unique - manual DB intervention required') for service_record in service_records: if service_record and (service_record['hostid'] != self.instance_id): raise Exception('service type (' + str(self.__service_name__) + ') already exists in system with different host_id - detail: my_host_id=' + str( self.instance_id) + ' db_host_id=' + str(service_record['hostid'])) # if all checks out, then add/update the registration ret = db_services.add(self.instance_id, self.__service_name__, service_template, session=dbsession) try: my_service_record = { 'hostid': self.instance_id, 'servicename': self.__service_name__, } my_service_record.update(service_template) servicestatus.set_my_service_record(my_service_record) self.service_record = my_service_record except Exception as err: logger.warn('could not set local service information - exception: {}'.format(str(err))) except Exception as err: raise err service_record = servicestatus.get_my_service_record() servicestatus.set_status(service_record, up=True, available=True, update_db=True, versions=self.versions) logger.info('Service registration complete') return True
def squash(unpackdir, cachedir, layers): rootfsdir = unpackdir + "/rootfs" if os.path.exists(unpackdir + "/squashed.tar"): return (True) if not os.path.exists(rootfsdir): os.makedirs(rootfsdir) revlayer = list(layers) revlayer.reverse() l_excludes = {} l_opqexcludes = { } # stores list of special files to exclude only for next layer (.wh..wh..opq handling) last_opqexcludes = {} # opq exlcudes for the last layer for l in revlayer: htype, layer = l.split(":", 1) layertar = get_layertarfile(unpackdir, cachedir, layer) count = 0 logger.debug("\tPass 1: " + str(layertar)) whpatt = re.compile(".*/\.wh\..*") whopqpatt = re.compile(".*/\.wh\.\.wh\.\.opq") l_opqexcludes[layer] = {} myexcludes = {} opqexcludes = {} tarfilenames = get_tar_filenames(layertar) for fname in tarfilenames: # checks for whiteout conditions if whopqpatt.match(fname): # found an opq entry, which means that this files in the next layer down (only) should not be included fsub = re.sub(r"\.wh\.\.wh\.\.opq", "", fname, 1) # never include the whiteout file itself myexcludes[fname] = True opqexcludes[fsub] = True elif whpatt.match(fname): # found a normal whiteout, which means that this file in any lower layer should be excluded fsub = re.sub(r"\.wh\.", "", fname, 1) # never include a whiteout file myexcludes[fname] = True myexcludes[fsub] = True else: # if the last processed layer had an opq whiteout, check file to see if it lives in the opq directory if last_opqexcludes: dtoks = fname.split("/") for i in range(0, len(dtoks)): dtok = '/'.join(dtoks[0:i]) dtokwtrail = '/'.join(dtoks[0:i]) + "/" if dtok in last_opqexcludes or dtokwtrail in last_opqexcludes: l_opqexcludes[layer][fname] = True break # build up the list of excludes as we move down the layers for l in l_excludes.keys(): myexcludes.update(l_excludes[l]) l_excludes[layer] = myexcludes last_opqexcludes.update(opqexcludes) logger.debug("Pass 3: untarring layers with exclusions") imageSize = 0 for l in layers: htype, layer = l.split(":", 1) layertar = get_layertarfile(unpackdir, cachedir, layer) imageSize = imageSize + os.path.getsize(layertar) # write out the exluded files, adding the per-layer excludes if present with open(unpackdir + "/efile", 'w') as OFH: for efile in l_excludes[layer]: OFH.write("%s\n" % efile) if layer in l_opqexcludes and l_opqexcludes[layer]: for efile in l_opqexcludes[layer]: logger.debug("adding special for layer exclude: " + str(efile)) OFH.write("%s\n" % efile) retry = True success = False last_err = None max_retries = 10 retries = 0 while (not success) and (retry): tarcmd = "tar -C " + rootfsdir + " -x -X " + unpackdir + "/efile -f " + layertar logger.debug("untarring squashed tarball: " + str(tarcmd)) try: rc, sout, serr = utils.run_command(tarcmd) if rc != 0: logger.debug("tar error encountered, attempting to handle") handled = handle_tar_error(tarcmd, rc, sout, serr, unpackdir=unpackdir, rootfsdir=rootfsdir, layer=layer, layertar=layertar) if not handled: raise Exception("command failed: cmd=" + str(tarcmd) + " exitcode=" + str(rc) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) else: logger.debug( "tar error successfully handled, retrying") else: logger.debug("command succeeded: stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) success = True except Exception as err: logger.error("command failed with exception - " + str(err)) last_err = err success = False retry = False # safety net if retries > max_retries: retry = False retries = retries + 1 if not success: if last_err: raise last_err else: raise Exception("unknown exception in untar") return ("done", imageSize)
def _sync_group(self, group_download_result: GroupDownloadResult, full_flush=False, local_repo=None, operation_id=None): """ Sync data from a single group and return the data. This operation is scoped to a transaction on the db. :param group_download_result :return: """ total_updated_count = 0 result = build_group_sync_result() result['group'] = group_download_result.group sync_started = None db = get_session() db.refresh(self.metadata) group_db_obj = self.group_by_name(group_download_result.group) if not group_db_obj: logger.error( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Skipping group sync. Record not found in db, should have been synced already' )) return result sync_started = time.time() download_started = group_download_result.started.replace( tzinfo=datetime.timezone.utc) try: updated_images = set( ) # To get unique set of all images updated by this sync if full_flush: logger.info( log_msg_ctx(operation_id, group_download_result.feed, group_download_result.group, 'Performing group data flush prior to sync')) self._flush_group(group_db_obj, operation_id=operation_id) mapper = self._load_mapper(group_db_obj) # Iterate thru the records and commit count = 0 for record in local_repo.read(group_download_result.feed, group_download_result.group, 0): mapped = mapper.map(record) updated_image_ids = self.update_vulnerability( db, mapped, vulnerability_processing_fn=VulnerabilityFeed. __vuln_processing_fn__) updated_images = updated_images.union( set(updated_image_ids )) # Record after commit to ensure in-sync. merged = db.merge(mapped) total_updated_count += 1 count += 1 if len(updated_image_ids) > 0: db.flush( ) # Flush after every one so that mem footprint stays small if lots of images are updated if count >= self.RECORDS_PER_CHUNK: # Commit group_db_obj.count = self.record_count( group_db_obj.name, db) db.commit() logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'DB Update Progress: {}/{}'.format( total_updated_count, group_download_result.total_records))) db = get_session() count = 0 else: group_db_obj.count = self.record_count(group_db_obj.name, db) db.commit() logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'DB Update Progress: {}/{}'.format( total_updated_count, group_download_result.total_records))) db = get_session() logger.debug( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Updating last sync timestamp to {}'.format( download_started))) group_db_obj = self.group_by_name(group_download_result.group) group_db_obj.last_sync = download_started group_db_obj.count = self.record_count(group_db_obj.name, db) db.add(group_db_obj) db.commit() except Exception as e: logger.exception( log_msg_ctx(operation_id, group_download_result.feed, group_download_result.group, 'Error syncing group')) db.rollback() raise e finally: total_group_time = time.time() - download_started.timestamp() sync_time = time.time() - sync_started logger.info( log_msg_ctx(operation_id, group_download_result.feed, group_download_result.group, 'Sync to db duration: {} sec'.format(sync_time))) logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Total sync, including download, duration: {} sec'.format( total_group_time))) result['updated_record_count'] = total_updated_count result['status'] = 'success' result['total_time_seconds'] = total_group_time result['updated_image_count'] = 0 return result