def get_orig(userId, bucket, archiveid):
global archive_initialized, data_volume, use_db
if not archive_initialized:
raise Exception("archive not initialized")
ret = {}
if use_db:
try:
with db.session_scope() as dbsession:
result = db_archivedocument.get(userId,
bucket,
archiveid,
session=dbsession)
if result and 'jsondata' in result:
ret = json.loads(result['jsondata'])
del result
else:
raise Exception("no archive record JSON data found in DB")
except Exception as err:
logger.debug("cannot get data: exception - " + str(err))
raise err
else:
try:
with open(os.path.join(data_volume, bucket, archiveid + ".json"),
'r') as FH:
ret = json.loads(FH.read())
except Exception as err:
logger.debug("cannot get data: exception - " + str(err))
raise err
return (ret)
def get(userId, bucket, archiveid):
global archive_initialized, data_volume, use_db
if not archive_initialized:
raise Exception("archive not initialized")
ret = {}
try:
with db.session_scope() as dbsession:
result = db_archivedocument.get(userId,
bucket,
archiveid,
session=dbsession)
if result:
if use_db:
if 'jsondata' in result:
ret = json.loads(result['jsondata'])
del result
else:
raise Exception("no archive record JSON data found in DB")
else:
ret = read_archive_file(userId, bucket, archiveid)
except Exception as err:
logger.debug("cannot get data: exception - " + str(err))
raise err
return (ret)
def db_upgrade_003_004():
engine = anchore_engine.db.entities.common.get_engine()
from anchore_engine.db import db_catalog_image, db_archivedocument, session_scope
import anchore_engine.common
newcolumns = [
Column('arch', String, primary_key=False),
Column('distro', String, primary_key=False),
Column('distro_version', String, primary_key=False),
Column('dockerfile_mode', String, primary_key=False),
Column('image_size', BigInteger, primary_key=False),
Column('layer_count', Integer, primary_key=False)
]
for column in newcolumns:
try:
table_name = 'catalog_image'
cn = column.compile(dialect=engine.dialect)
ct = column.type.compile(engine.dialect)
engine.execute('ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s' % (table_name, cn, ct))
except Exception as e:
log.err('failed to perform DB upgrade on catalog_image adding column - exception: {}'.format(str(e)))
raise Exception('failed to perform DB upgrade on catalog_image adding column - exception: {}'.format(str(e)))
with session_scope() as dbsession:
image_records = db_catalog_image.get_all(session=dbsession)
for image_record in image_records:
userId = image_record['userId']
imageDigest = image_record['imageDigest']
log.err("upgrade: processing image " + str(imageDigest) + " : " + str(userId))
try:
# get the image analysis data from archive
image_data = None
with session_scope() as dbsession:
result = db_archivedocument.get(userId, 'analysis_data', imageDigest, session=dbsession)
if result and 'jsondata' in result:
image_data = json.loads(result['jsondata'])['document']
if image_data:
# update the record and store
anchore_engine.common.helpers.update_image_record_with_analysis_data(image_record, image_data)
with session_scope() as dbsession:
db_catalog_image.update_record(image_record, session=dbsession)
else:
raise Exception("upgrade: no analysis data found in archive for image: " + str(imageDigest))
except Exception as err:
log.err("upgrade: failed to populate new columns with existing data for image (" + str(imageDigest) + "), record may be incomplete: " + str(err))
return True
def get_by_uri(self, uri: str) -> bytes:
userId, bucket, key = self._parse_uri(uri)
try:
with db.session_scope() as dbsession:
result = db_archivedocument.get(userId, bucket, key, session=dbsession)
if result:
return utils.ensure_bytes(self._decode(result))
else:
raise ObjectKeyNotFoundError(userId, bucket, key, caused_by=None)
except Exception as err:
logger.debug("cannot get data: exception - " + str(err))
raise err
def initialize():
global archive_initialized, data_volume, use_db, archive_driver
localconfig = anchore_engine.configuration.localconfig.get_config()
myconfig = localconfig['services']['catalog']
try:
data_volume = None
if 'archive_data_dir' in myconfig:
data_volume = myconfig['archive_data_dir']
archive_driver = 'db'
if 'archive_driver' in myconfig:
archive_driver = myconfig['archive_driver']
if 'use_db' in myconfig and myconfig['use_db']:
archive_driver = 'db'
# driver specific initializations here
if archive_driver == 'db':
use_db = True
else:
use_db = False
initialize_archive_file(myconfig)
except Exception as err:
raise err
logger.debug("archive initialization config: " +
str([archive_driver, use_db, data_volume]))
# this section is for conversion on initialization between db driver and other driver
with db.session_scope() as dbsession:
logger.debug("running archive driver converter")
if use_db:
# need to check if any archive records do not have the document field populated, and if so try to import from localfs
dbfilter = {'jsondata': '{}'}
archive_matches = db_archivedocument.list_all(session=dbsession,
**dbfilter)
for archive_match in archive_matches:
userId = archive_match['userId']
bucket = archive_match['bucket']
archiveid = archive_match['archiveId']
try:
fs_data = read_archive_file(userId,
bucket,
archiveid,
driver_override='localfs')
except Exception as err:
logger.debug("no data: " + str(err))
fs_data = None
if fs_data:
logger.debug("document data - converting driver->DB: " +
str([userId, bucket, archiveid]))
with db.session_scope() as subdbsession:
db_archivedocument.add(
userId,
bucket,
archiveid,
archiveid + ".json",
{'jsondata': json.dumps(fs_data)},
session=subdbsession)
delete_archive_file(userId,
bucket,
archiveid,
driver_override='localfs')
else:
# need to check if any archive records DO have the document field populated, and if so try to export to localfs
archive_matches = db_archivedocument.list_all_notempty(
session=dbsession)
for archive_match in archive_matches:
userId = archive_match['userId']
bucket = archive_match['bucket']
archiveid = archive_match['archiveId']
archive_record = db_archivedocument.get(userId,
bucket,
archiveid,
session=dbsession)
db_data = json.loads(archive_record['jsondata'])
logger.debug("document data - converting DB->driver: " +
str([userId, bucket, archiveid]))
dataref = write_archive_file(userId,
bucket,
archiveid,
db_data,
driver_override='localfs')
with db.session_scope() as subdbsession:
db_archivedocument.add(userId,
bucket,
archiveid,
archiveid + ".json",
{'jsondata': "{}"},
session=subdbsession)
if False:
for archive_record in db_archivedocument.get_all_iter(
session=dbsession):
userId = archive_record['userId']
bucket = archive_record['bucket']
archiveid = archive_record['archiveId']
dataref = archive_record['documentName']
if archive_record['jsondata']:
if archive_record['jsondata'] == "{}":
db_data = None
else:
try:
db_data = json.loads(archive_record['jsondata'])
except:
logger.warn(
"could no load jsondata for archive record: " +
str([userId, bucket, archiveid]))
db_data = None
else:
db_data = None
if use_db and not db_data:
try:
fs_data = read_archive_file(userId,
bucket,
archiveid,
driver_override='localfs')
except Exception as err:
logger.debug("no data: " + str(err))
fs_data = None
if fs_data:
logger.debug(
"document data not in DB but is on FS - converting: "
+ str([userId, bucket, archiveid]))
with db.session_scope() as subdbsession:
db_archivedocument.add(
userId,
bucket,
archiveid,
archiveid + ".json",
{'jsondata': json.dumps(fs_data)},
session=subdbsession)
delete_archive_file(userId,
bucket,
archiveid,
driver_override='localfs')
elif not use_db and db_data:
logger.debug(
"document data not on FS but is in DB - converting: " +
str([userId, bucket, archiveid]))
dataref = write_archive_file(userId,
bucket,
archiveid,
db_data,
driver_override='localfs')
with db.session_scope() as subdbsession:
db_archivedocument.add(userId,
bucket,
archiveid,
archiveid + ".json",
{'jsondata': "{}"},
session=subdbsession)
logger.debug("archive driver converter complete")
archive_initialized = True
return (True)
