def _register(self): if not self.is_enabled: logger.error( 'Service not enabled in config, not registering service: ' + self.name) raise Exception('No service enabled, cannot continue bootstrap') logger.info('Registering service: {}'.format(self.name)) service_template = { 'type': 'anchore', 'base_url': 'N/A', 'status_base_url': 'N/A', 'version': 'v1', 'short_description': '' } hstring = 'http' if 'external_tls' in self.configuration: if self.configuration.get('external_tls', False): hstring = 'https' elif 'ssl_enable' in self.configuration: if self.configuration.get('ssl_enable', False): hstring = 'https' endpoint_hostname = endpoint_port = endpoint_hostport = None if self.configuration.get('external_hostname', False): endpoint_hostname = self.configuration.get('external_hostname') elif self.configuration.get('endpoint_hostname', False): endpoint_hostname = self.configuration.get('endpoint_hostname') if self.configuration.get('external_port', False): endpoint_port = int(self.configuration.get('external_port')) elif self.configuration.get('port', False): endpoint_port = int(self.configuration.get('port')) if endpoint_hostname: endpoint_hostport = endpoint_hostname if endpoint_port: endpoint_hostport = endpoint_hostport + ":" + str( endpoint_port) if endpoint_hostport: service_template['base_url'] = "{}://{}".format( hstring, endpoint_hostport) else: raise Exception( "could not construct service base_url - please check service configuration for hostname/port settings" ) try: service_template['status'] = False service_template['status_message'] = taskstate.base_state( 'service_status') with session_scope() as dbsession: service_records = db_services.get_byname(self.__service_name__, session=dbsession) # fail if trying to add a service that must be unique in the system, but one already is registered in DB if self.__is_unique_service__: if len(service_records) > 1: raise Exception( 'more than one entry for service type (' + str(self.__service_name__) + ') exists in DB, but service must be unique - manual DB intervention required' ) for service_record in service_records: if service_record and (service_record['hostid'] != self.instance_id): raise Exception( 'service type (' + str(self.__service_name__) + ') already exists in system with different host_id - detail: my_host_id=' + str(self.instance_id) + ' db_host_id=' + str(service_record['hostid'])) # if all checks out, then add/update the registration ret = db_services.add(self.instance_id, self.__service_name__, service_template, session=dbsession) try: my_service_record = { 'hostid': self.instance_id, 'servicename': self.__service_name__, } my_service_record.update(service_template) servicestatus.set_my_service_record(my_service_record) self.service_record = my_service_record except Exception as err: logger.warn( 'could not set local service information - exception: {}' .format(str(err))) except Exception as err: raise err service_record = servicestatus.get_my_service_record() servicestatus.set_status(service_record, up=True, available=True, update_db=True, versions=self.versions) logger.info('Service registration complete') return True
def import_image(operation_id, account, import_manifest: InternalImportManifest): """ The main thread of exec for importing an image :param operation_id: :param account: :param import_manifest: :return: """ timer = int(time.time()) analysis_events = [] config = localconfig.get_config() all_content_types = config.get("image_content_types", []) + config.get( "image_metadata_types", []) image_digest = import_manifest.digest try: catalog_client = internal_client_for(CatalogClient, account) # check to make sure image is still in DB catalog_client = internal_client_for(CatalogClient, account) try: image_record = catalog_client.get_image(image_digest) if not image_record: raise Exception("empty image record from catalog") except Exception as err: logger.debug_exception("Could not get image record") logger.warn( "dequeued image cannot be fetched from catalog - skipping analysis (" + str(image_digest) + ") - exception: " + str(err)) return True if image_record["analysis_status"] != taskstate.base_state("analyze"): logger.info( "dequeued image to import is not in base 'not_analyzed' state - skipping import" ) return True try: last_analysis_status = image_record["analysis_status"] image_record = update_analysis_started(catalog_client, image_digest, image_record) logger.info("Loading content from import") sbom_map = get_content(import_manifest, catalog_client) manifest = sbom_map.get("manifest") try: logger.info("processing image import data") image_data, analysis_manifest = process_import( image_record, sbom_map, import_manifest) except AnchoreException as e: event = events.ImageAnalysisFailed(user_id=account, image_digest=image_digest, error=e.to_dict()) analysis_events.append(event) raise # Store the manifest in the object store logger.info("storing image manifest") catalog_client.put_document(bucket="manifest_data", name=image_digest, inobj=json.dumps(manifest)) # Save the results to the upstream components and data stores logger.info("storing import result") store_analysis_results( account, image_digest, image_record, image_data, manifest, analysis_events, all_content_types, ) logger.info("updating image catalog record analysis_status") last_analysis_status = image_record["analysis_status"] image_record = update_analysis_complete(catalog_client, image_digest, image_record) try: analysis_events.extend( notify_analysis_complete(image_record, last_analysis_status)) except Exception as err: logger.warn( "failed to enqueue notification on image analysis state update - exception: " + str(err)) logger.info("analysis complete: " + str(account) + " : " + str(image_digest)) try: catalog_client.update_image_import_status(operation_id, status="complete") except Exception as err: logger.debug_exception( "failed updating import status success, will continue and rely on expiration for GC later" ) try: metrics.counter_inc(name="anchore_import_success") run_time = float(time.time() - timer) metrics.histogram_observe( "anchore_import_time_seconds", run_time, buckets=IMPORT_TIME_SECONDS_BUCKETS, status="success", ) except Exception as err: logger.warn(str(err)) except Exception as err: run_time = float(time.time() - timer) logger.exception("problem importing image - exception: " + str(err)) analysis_failed_metrics(run_time) # Transition the image record to failure status image_record = update_analysis_failed(catalog_client, image_digest, image_record) try: catalog_client.update_image_import_status(operation_id, status="failed") except Exception as err: logger.debug_exception( "failed updating import status failure, will continue and rely on expiration for GC later" ) if account and image_digest: for image_detail in image_record["image_detail"]: fulltag = (image_detail["registry"] + "/" + image_detail["repo"] + ":" + image_detail["tag"]) event = events.UserAnalyzeImageFailed(user_id=account, full_tag=fulltag, error=str(err)) analysis_events.append(event) finally: if analysis_events: emit_events(catalog_client, analysis_events) except Exception as err: logger.debug_exception("Could not import image") logger.warn("job processing bailed - exception: " + str(err)) raise err return True
def _register(self): if not self.is_enabled: logger.error( "Service not enabled in config, not registering service: " + self.name) raise Exception("No service enabled, cannot continue bootstrap") logger.info("Registering service: {}".format(self.name)) service_template = { "type": "anchore", "base_url": "N/A", "status_base_url": "N/A", "version": "v1", "short_description": "", } hstring = "http" if "external_tls" in self.configuration: if self.configuration.get("external_tls", False): hstring = "https" elif "ssl_enable" in self.configuration: if self.configuration.get("ssl_enable", False): hstring = "https" endpoint_hostname = endpoint_port = endpoint_hostport = None if self.configuration.get("external_hostname", False): endpoint_hostname = self.configuration.get("external_hostname") elif self.configuration.get("endpoint_hostname", False): endpoint_hostname = self.configuration.get("endpoint_hostname") if self.configuration.get("external_port", False): endpoint_port = int(self.configuration.get("external_port")) elif self.configuration.get("port", False): endpoint_port = int(self.configuration.get("port")) if endpoint_hostname: endpoint_hostport = endpoint_hostname if endpoint_port: endpoint_hostport = endpoint_hostport + ":" + str( endpoint_port) if endpoint_hostport: service_template["base_url"] = "{}://{}".format( hstring, endpoint_hostport) else: raise Exception( "could not construct service base_url - please check service configuration for hostname/port settings" ) try: service_template["status"] = False service_template["status_message"] = taskstate.base_state( "service_status") with session_scope() as dbsession: service_records = db_services.get_byname(self.__service_name__, session=dbsession) # fail if trying to add a service that must be unique in the system, but one already is registered in DB if self.__is_unique_service__: if len(service_records) > 1: raise Exception( "more than one entry for service type (" + str(self.__service_name__) + ") exists in DB, but service must be unique - manual DB intervention required" ) for service_record in service_records: if service_record and (service_record["hostid"] != self.instance_id): raise Exception( "service type (" + str(self.__service_name__) + ") already exists in system with different host_id - detail: my_host_id=" + str(self.instance_id) + " db_host_id=" + str(service_record["hostid"])) # if all checks out, then add/update the registration ret = db_services.add( self.instance_id, self.__service_name__, service_template, session=dbsession, ) try: my_service_record = { "hostid": self.instance_id, "servicename": self.__service_name__, } my_service_record.update(service_template) servicestatus.set_my_service_record(my_service_record) self.service_record = my_service_record except Exception as err: logger.warn( "could not set local service information - exception: {}" .format(str(err))) except Exception as err: raise err service_record = servicestatus.get_my_service_record() servicestatus.set_status( service_record, up=True, available=True, update_db=True, versions=self.versions, ) logger.info("Service registration complete") return True
def registerService(sname, config, enforce_unique=True): ret = False myconfig = config['services'][sname] # TODO add version support/detection here service_template = { 'type': 'anchore', 'base_url': 'N/A', 'status_base_url': 'N/A', 'version': 'v1', 'short_description': '' } #if 'ssl_enable' in myconfig and myconfig['ssl_enable']: if myconfig.get('ssl_enable', False) or myconfig.get( 'external_tls', False): hstring = "https" else: hstring = "http" endpoint_hostname = endpoint_port = endpoint_hostport = None if 'endpoint_hostname' in myconfig: endpoint_hostname = myconfig['endpoint_hostname'] service_template[ 'base_url'] = hstring + "://" + myconfig['endpoint_hostname'] if 'port' in myconfig: endpoint_port = int(myconfig['port']) service_template['base_url'] += ":" + str(endpoint_port) if endpoint_hostname: endpoint_hostport = endpoint_hostname if endpoint_port: endpoint_hostport = endpoint_hostport + ":" + str(endpoint_port) try: service_template['status'] = False service_template['status_message'] = taskstate.base_state( 'service_status') with session_scope() as dbsession: service_records = db_services.get_byname(sname, session=dbsession) # fail if trying to add a service that must be unique in the system, but one already is registered in DB if enforce_unique: if len(service_records) > 1: raise Exception( "more than one entry for service type (" + str(sname) + ") exists in DB, but service must be unique - manual DB intervention required" ) for service_record in service_records: if service_record and (service_record['hostid'] != config['host_id']): raise Exception( "service type (" + str(sname) + ") already exists in system with different host_id - detail: my_host_id=" + str(config['host_id']) + " db_host_id=" + str(service_record['hostid'])) # if all checks out, then add/update the registration ret = db_services.add(config['host_id'], sname, service_template, session=dbsession) try: my_service_record = { 'hostid': config['host_id'], 'servicename': sname, } my_service_record.update(service_template) servicestatus.set_my_service_record(my_service_record) except Exception as err: logger.warn( "could not set local service information - exception: {}". format(str(err))) except Exception as err: raise err return (ret)
def registerService(sname, config, enforce_unique=True): ret = False myconfig = config['services'][sname] # TODO add version support/detection here service_template = { 'type': 'anchore', 'base_url': 'N/A', 'status_base_url': 'N/A', 'version': 'v1', 'short_description': '' } if 'ssl_enable' in myconfig and myconfig['ssl_enable']: hstring = "https" else: hstring = "http" endpoint_hostname = endpoint_port = endpoint_hostport = None if 'endpoint_hostname' in myconfig: endpoint_hostname = myconfig['endpoint_hostname'] service_template[ 'base_url'] = hstring + "://" + myconfig['endpoint_hostname'] if 'port' in myconfig: endpoint_port = int(myconfig['port']) service_template['base_url'] += ":" + str(endpoint_port) if endpoint_hostname: endpoint_hostport = endpoint_hostname if endpoint_port: endpoint_hostport = endpoint_hostport + ":" + str(endpoint_port) try: service_template['status'] = False service_template['status_message'] = taskstate.base_state( 'service_status') with session_scope() as dbsession: service_records = db_services.get_byname(sname, session=dbsession) # fail if trying to add a service that must be unique in the system, but one already is registered in DB if enforce_unique: if len(service_records) > 1: raise Exception( "more than one entry for service type (" + str(sname) + ") exists in DB, but service must be unique - manual DB intervention required" ) for service_record in service_records: if service_record and (service_record['hostid'] != config['host_id']): raise Exception( "service type (" + str(sname) + ") already exists in system with different host_id - detail: my_host_id=" + str(config['host_id']) + " db_host_id=" + str(service_record['hostid'])) # in any case, check if another host is registered that has the same endpoint #for service_record in service_records: # if service_record['base_url'] and service_record['base_url'] != 'N/A': # service_hostport = re.sub("^http.//", "", service_record['base_url']) # # if a different host_id has the same endpoint, fail # if (service_hostport == endpoint_hostport) and (config['host_id'] != service_record['hostid']): # raise Exception("trying to add new host but found conflicting endpoint from another host in DB - detail: my_host_id=" + str(config['host_id']) + " db_host_id="+str(service_record['hostid'])+" my_host_endpoint="+str(endpoint_hostport)+" db_host_endpoint="+str(service_hostport)) # if all checks out, then add/update the registration ret = db_services.add(config['host_id'], sname, service_template, session=dbsession) except Exception as err: raise err return (ret)