def _register(self):
if not self.is_enabled:
logger.error(
'Service not enabled in config, not registering service: ' +
self.name)
raise Exception('No service enabled, cannot continue bootstrap')
logger.info('Registering service: {}'.format(self.name))
service_template = {
'type': 'anchore',
'base_url': 'N/A',
'status_base_url': 'N/A',
'version': 'v1',
'short_description': ''
}
hstring = 'http'
if 'external_tls' in self.configuration:
if self.configuration.get('external_tls', False):
hstring = 'https'
elif 'ssl_enable' in self.configuration:
if self.configuration.get('ssl_enable', False):
hstring = 'https'
endpoint_hostname = endpoint_port = endpoint_hostport = None
if self.configuration.get('external_hostname', False):
endpoint_hostname = self.configuration.get('external_hostname')
elif self.configuration.get('endpoint_hostname', False):
endpoint_hostname = self.configuration.get('endpoint_hostname')
if self.configuration.get('external_port', False):
endpoint_port = int(self.configuration.get('external_port'))
elif self.configuration.get('port', False):
endpoint_port = int(self.configuration.get('port'))
if endpoint_hostname:
endpoint_hostport = endpoint_hostname
if endpoint_port:
endpoint_hostport = endpoint_hostport + ":" + str(
endpoint_port)
if endpoint_hostport:
service_template['base_url'] = "{}://{}".format(
hstring, endpoint_hostport)
else:
raise Exception(
"could not construct service base_url - please check service configuration for hostname/port settings"
)
try:
service_template['status'] = False
service_template['status_message'] = taskstate.base_state(
'service_status')
with session_scope() as dbsession:
service_records = db_services.get_byname(self.__service_name__,
session=dbsession)
# fail if trying to add a service that must be unique in the system, but one already is registered in DB
if self.__is_unique_service__:
if len(service_records) > 1:
raise Exception(
'more than one entry for service type (' +
str(self.__service_name__) +
') exists in DB, but service must be unique - manual DB intervention required'
)
for service_record in service_records:
if service_record and (service_record['hostid'] !=
self.instance_id):
raise Exception(
'service type (' + str(self.__service_name__) +
') already exists in system with different host_id - detail: my_host_id='
+ str(self.instance_id) + ' db_host_id=' +
str(service_record['hostid']))
# if all checks out, then add/update the registration
ret = db_services.add(self.instance_id,
self.__service_name__,
service_template,
session=dbsession)
try:
my_service_record = {
'hostid': self.instance_id,
'servicename': self.__service_name__,
}
my_service_record.update(service_template)
servicestatus.set_my_service_record(my_service_record)
self.service_record = my_service_record
except Exception as err:
logger.warn(
'could not set local service information - exception: {}'
.format(str(err)))
except Exception as err:
raise err
service_record = servicestatus.get_my_service_record()
servicestatus.set_status(service_record,
up=True,
available=True,
update_db=True,
versions=self.versions)
logger.info('Service registration complete')
return True
def import_image(operation_id, account,
import_manifest: InternalImportManifest):
"""
The main thread of exec for importing an image
:param operation_id:
:param account:
:param import_manifest:
:return:
"""
timer = int(time.time())
analysis_events = []
config = localconfig.get_config()
all_content_types = config.get("image_content_types", []) + config.get(
"image_metadata_types", [])
image_digest = import_manifest.digest
try:
catalog_client = internal_client_for(CatalogClient, account)
# check to make sure image is still in DB
catalog_client = internal_client_for(CatalogClient, account)
try:
image_record = catalog_client.get_image(image_digest)
if not image_record:
raise Exception("empty image record from catalog")
except Exception as err:
logger.debug_exception("Could not get image record")
logger.warn(
"dequeued image cannot be fetched from catalog - skipping analysis ("
+ str(image_digest) + ") - exception: " + str(err))
return True
if image_record["analysis_status"] != taskstate.base_state("analyze"):
logger.info(
"dequeued image to import is not in base 'not_analyzed' state - skipping import"
)
return True
try:
last_analysis_status = image_record["analysis_status"]
image_record = update_analysis_started(catalog_client,
image_digest, image_record)
logger.info("Loading content from import")
sbom_map = get_content(import_manifest, catalog_client)
manifest = sbom_map.get("manifest")
try:
logger.info("processing image import data")
image_data, analysis_manifest = process_import(
image_record, sbom_map, import_manifest)
except AnchoreException as e:
event = events.ImageAnalysisFailed(user_id=account,
image_digest=image_digest,
error=e.to_dict())
analysis_events.append(event)
raise
# Store the manifest in the object store
logger.info("storing image manifest")
catalog_client.put_document(bucket="manifest_data",
name=image_digest,
inobj=json.dumps(manifest))
# Save the results to the upstream components and data stores
logger.info("storing import result")
store_analysis_results(
account,
image_digest,
image_record,
image_data,
manifest,
analysis_events,
all_content_types,
)
logger.info("updating image catalog record analysis_status")
last_analysis_status = image_record["analysis_status"]
image_record = update_analysis_complete(catalog_client,
image_digest, image_record)
try:
analysis_events.extend(
notify_analysis_complete(image_record,
last_analysis_status))
except Exception as err:
logger.warn(
"failed to enqueue notification on image analysis state update - exception: "
+ str(err))
logger.info("analysis complete: " + str(account) + " : " +
str(image_digest))
try:
catalog_client.update_image_import_status(operation_id,
status="complete")
except Exception as err:
logger.debug_exception(
"failed updating import status success, will continue and rely on expiration for GC later"
)
try:
metrics.counter_inc(name="anchore_import_success")
run_time = float(time.time() - timer)
metrics.histogram_observe(
"anchore_import_time_seconds",
run_time,
buckets=IMPORT_TIME_SECONDS_BUCKETS,
status="success",
)
except Exception as err:
logger.warn(str(err))
except Exception as err:
run_time = float(time.time() - timer)
logger.exception("problem importing image - exception: " +
str(err))
analysis_failed_metrics(run_time)
# Transition the image record to failure status
image_record = update_analysis_failed(catalog_client, image_digest,
image_record)
try:
catalog_client.update_image_import_status(operation_id,
status="failed")
except Exception as err:
logger.debug_exception(
"failed updating import status failure, will continue and rely on expiration for GC later"
)
if account and image_digest:
for image_detail in image_record["image_detail"]:
fulltag = (image_detail["registry"] + "/" +
image_detail["repo"] + ":" +
image_detail["tag"])
event = events.UserAnalyzeImageFailed(user_id=account,
full_tag=fulltag,
error=str(err))
analysis_events.append(event)
finally:
if analysis_events:
emit_events(catalog_client, analysis_events)
except Exception as err:
logger.debug_exception("Could not import image")
logger.warn("job processing bailed - exception: " + str(err))
raise err
return True
def _register(self):
if not self.is_enabled:
logger.error(
"Service not enabled in config, not registering service: " +
self.name)
raise Exception("No service enabled, cannot continue bootstrap")
logger.info("Registering service: {}".format(self.name))
service_template = {
"type": "anchore",
"base_url": "N/A",
"status_base_url": "N/A",
"version": "v1",
"short_description": "",
}
hstring = "http"
if "external_tls" in self.configuration:
if self.configuration.get("external_tls", False):
hstring = "https"
elif "ssl_enable" in self.configuration:
if self.configuration.get("ssl_enable", False):
hstring = "https"
endpoint_hostname = endpoint_port = endpoint_hostport = None
if self.configuration.get("external_hostname", False):
endpoint_hostname = self.configuration.get("external_hostname")
elif self.configuration.get("endpoint_hostname", False):
endpoint_hostname = self.configuration.get("endpoint_hostname")
if self.configuration.get("external_port", False):
endpoint_port = int(self.configuration.get("external_port"))
elif self.configuration.get("port", False):
endpoint_port = int(self.configuration.get("port"))
if endpoint_hostname:
endpoint_hostport = endpoint_hostname
if endpoint_port:
endpoint_hostport = endpoint_hostport + ":" + str(
endpoint_port)
if endpoint_hostport:
service_template["base_url"] = "{}://{}".format(
hstring, endpoint_hostport)
else:
raise Exception(
"could not construct service base_url - please check service configuration for hostname/port settings"
)
try:
service_template["status"] = False
service_template["status_message"] = taskstate.base_state(
"service_status")
with session_scope() as dbsession:
service_records = db_services.get_byname(self.__service_name__,
session=dbsession)
# fail if trying to add a service that must be unique in the system, but one already is registered in DB
if self.__is_unique_service__:
if len(service_records) > 1:
raise Exception(
"more than one entry for service type (" +
str(self.__service_name__) +
") exists in DB, but service must be unique - manual DB intervention required"
)
for service_record in service_records:
if service_record and (service_record["hostid"] !=
self.instance_id):
raise Exception(
"service type (" + str(self.__service_name__) +
") already exists in system with different host_id - detail: my_host_id="
+ str(self.instance_id) + " db_host_id=" +
str(service_record["hostid"]))
# if all checks out, then add/update the registration
ret = db_services.add(
self.instance_id,
self.__service_name__,
service_template,
session=dbsession,
)
try:
my_service_record = {
"hostid": self.instance_id,
"servicename": self.__service_name__,
}
my_service_record.update(service_template)
servicestatus.set_my_service_record(my_service_record)
self.service_record = my_service_record
except Exception as err:
logger.warn(
"could not set local service information - exception: {}"
.format(str(err)))
except Exception as err:
raise err
service_record = servicestatus.get_my_service_record()
servicestatus.set_status(
service_record,
up=True,
available=True,
update_db=True,
versions=self.versions,
)
logger.info("Service registration complete")
return True
def registerService(sname, config, enforce_unique=True):
ret = False
myconfig = config['services'][sname]
# TODO add version support/detection here
service_template = {
'type': 'anchore',
'base_url': 'N/A',
'status_base_url': 'N/A',
'version': 'v1',
'short_description': ''
}
#if 'ssl_enable' in myconfig and myconfig['ssl_enable']:
if myconfig.get('ssl_enable', False) or myconfig.get(
'external_tls', False):
hstring = "https"
else:
hstring = "http"
endpoint_hostname = endpoint_port = endpoint_hostport = None
if 'endpoint_hostname' in myconfig:
endpoint_hostname = myconfig['endpoint_hostname']
service_template[
'base_url'] = hstring + "://" + myconfig['endpoint_hostname']
if 'port' in myconfig:
endpoint_port = int(myconfig['port'])
service_template['base_url'] += ":" + str(endpoint_port)
if endpoint_hostname:
endpoint_hostport = endpoint_hostname
if endpoint_port:
endpoint_hostport = endpoint_hostport + ":" + str(endpoint_port)
try:
service_template['status'] = False
service_template['status_message'] = taskstate.base_state(
'service_status')
with session_scope() as dbsession:
service_records = db_services.get_byname(sname, session=dbsession)
# fail if trying to add a service that must be unique in the system, but one already is registered in DB
if enforce_unique:
if len(service_records) > 1:
raise Exception(
"more than one entry for service type (" + str(sname) +
") exists in DB, but service must be unique - manual DB intervention required"
)
for service_record in service_records:
if service_record and (service_record['hostid'] !=
config['host_id']):
raise Exception(
"service type (" + str(sname) +
") already exists in system with different host_id - detail: my_host_id="
+ str(config['host_id']) + " db_host_id=" +
str(service_record['hostid']))
# if all checks out, then add/update the registration
ret = db_services.add(config['host_id'],
sname,
service_template,
session=dbsession)
try:
my_service_record = {
'hostid': config['host_id'],
'servicename': sname,
}
my_service_record.update(service_template)
servicestatus.set_my_service_record(my_service_record)
except Exception as err:
logger.warn(
"could not set local service information - exception: {}".
format(str(err)))
except Exception as err:
raise err
return (ret)
def registerService(sname, config, enforce_unique=True):
ret = False
myconfig = config['services'][sname]
# TODO add version support/detection here
service_template = {
'type': 'anchore',
'base_url': 'N/A',
'status_base_url': 'N/A',
'version': 'v1',
'short_description': ''
}
if 'ssl_enable' in myconfig and myconfig['ssl_enable']:
hstring = "https"
else:
hstring = "http"
endpoint_hostname = endpoint_port = endpoint_hostport = None
if 'endpoint_hostname' in myconfig:
endpoint_hostname = myconfig['endpoint_hostname']
service_template[
'base_url'] = hstring + "://" + myconfig['endpoint_hostname']
if 'port' in myconfig:
endpoint_port = int(myconfig['port'])
service_template['base_url'] += ":" + str(endpoint_port)
if endpoint_hostname:
endpoint_hostport = endpoint_hostname
if endpoint_port:
endpoint_hostport = endpoint_hostport + ":" + str(endpoint_port)
try:
service_template['status'] = False
service_template['status_message'] = taskstate.base_state(
'service_status')
with session_scope() as dbsession:
service_records = db_services.get_byname(sname, session=dbsession)
# fail if trying to add a service that must be unique in the system, but one already is registered in DB
if enforce_unique:
if len(service_records) > 1:
raise Exception(
"more than one entry for service type (" + str(sname) +
") exists in DB, but service must be unique - manual DB intervention required"
)
for service_record in service_records:
if service_record and (service_record['hostid'] !=
config['host_id']):
raise Exception(
"service type (" + str(sname) +
") already exists in system with different host_id - detail: my_host_id="
+ str(config['host_id']) + " db_host_id=" +
str(service_record['hostid']))
# in any case, check if another host is registered that has the same endpoint
#for service_record in service_records:
# if service_record['base_url'] and service_record['base_url'] != 'N/A':
# service_hostport = re.sub("^http.//", "", service_record['base_url'])
# # if a different host_id has the same endpoint, fail
# if (service_hostport == endpoint_hostport) and (config['host_id'] != service_record['hostid']):
# raise Exception("trying to add new host but found conflicting endpoint from another host in DB - detail: my_host_id=" + str(config['host_id']) + " db_host_id="+str(service_record['hostid'])+" my_host_endpoint="+str(endpoint_hostport)+" db_host_endpoint="+str(service_hostport))
# if all checks out, then add/update the registration
ret = db_services.add(config['host_id'],
sname,
service_template,
session=dbsession)
except Exception as err:
raise err
return (ret)
