def get(self, inventory_id):
"""Get inventory metadata by id.
Args:
inventory_id (str): Id of the inventory.
Returns:
object: Inventory metadata
"""
with self.config.scoped_session() as session:
result = DataAccess.get(session, inventory_id)
return result
def run(self):
"""Runs the import.
Raises:
NotImplementedError: If the importer encounters an unknown
inventory type.
"""
autocommit = self.session.autocommit
autoflush = self.session.autoflush
try:
self.session.autocommit = False
self.session.autoflush = True
root = DataAccess.get_root(self.readonly_session,
self.inventory_index_id)
inventory_index = DataAccess.get(self.readonly_session,
self.inventory_index_id)
description = {
'source': 'inventory',
'source_info': {
'inventory_index_id': self.inventory_index_id},
'source_root': self._type_name(root),
'pristine': True,
'gsuite_enabled': DataAccess.type_exists(
self.readonly_session,
self.inventory_index_id,
['gsuite_group', 'gsuite_user'])
}
LOGGER.debug('Model description: %s', description)
self.model.add_description(json.dumps(description,
sort_keys=True))
if root.get_resource_type() in ['organization']:
LOGGER.debug('Root resource is organization: %s', root)
else:
LOGGER.debug('Root resource is not organization: %s.', root)
item_counter = 0
LOGGER.debug('Start storing resources into models.')
for resource in DataAccess.iter(self.readonly_session,
self.inventory_index_id,
GCP_TYPE_LIST):
item_counter += 1
self._store_resource(resource)
if not item_counter % 1000:
# Flush database every 1000 resources
LOGGER.debug('Flushing model write session: %s',
item_counter)
self._flush_session()
if not item_counter % 100000:
# Commit every 100k resources while iterating
# through all the resources.
LOGGER.debug('Commiting model write session: %s',
item_counter)
self._commit_session()
self._commit_session()
LOGGER.debug('Finished storing resources into models.')
item_counter += self.model_action_wrapper(
DataAccess.iter(self.readonly_session,
self.inventory_index_id,
['role']),
self._convert_role
)
item_counter += self.model_action_wrapper(
DataAccess.iter(self.readonly_session,
self.inventory_index_id,
GCP_TYPE_LIST,
fetch_category=Categories.dataset_policy),
self._convert_dataset_policy
)
item_counter += self.model_action_wrapper(
DataAccess.iter(self.readonly_session,
self.inventory_index_id,
GCP_TYPE_LIST,
fetch_category=Categories.gcs_policy),
self._convert_gcs_policy
)
item_counter += self.model_action_wrapper(
DataAccess.iter(
self.readonly_session,
self.inventory_index_id,
GCP_TYPE_LIST,
fetch_category=Categories.kubernetes_service_config),
self._convert_service_config
)
self.model_action_wrapper(
DataAccess.iter(self.readonly_session,
self.inventory_index_id,
GSUITE_TYPE_LIST),
self._store_gsuite_principal
)
self.model_action_wrapper(
DataAccess.iter(self.readonly_session,
self.inventory_index_id,
GCP_TYPE_LIST,
fetch_category=Categories.enabled_apis),
self._convert_enabled_apis
)
self.model_action_wrapper(
DataAccess.iter(self.readonly_session,
self.inventory_index_id,
MEMBER_TYPE_LIST,
with_parent=True),
self._store_gsuite_membership,
post_action=self._store_gsuite_membership_post
)
self.model_action_wrapper(
DataAccess.iter(self.readonly_session,
self.inventory_index_id,
GROUPS_SETTINGS_LIST),
self._store_groups_settings
)
self.dao.denorm_group_in_group(self.session)
self.model_action_wrapper(
DataAccess.iter(self.readonly_session,
self.inventory_index_id,
GCP_TYPE_LIST,
fetch_category=Categories.iam_policy),
self._store_iam_policy
)
self.dao.expand_special_members(self.session)
except Exception as e: # pylint: disable=broad-except
LOGGER.exception(e)
buf = StringIO()
traceback.print_exc(file=buf)
buf.seek(0)
message = buf.read()
LOGGER.debug('Importer has an exception: %s', message)
self.model.set_error(message)
else:
LOGGER.debug('Set model status.')
for row in inventory_index.warning_messages:
self.model.add_warning('{}: {}'.format(row.resource_full_name,
row.warning_message))
self.model.set_done(item_counter)
finally:
LOGGER.debug('Finished running importer.')
self.session.commit()
self.session.autocommit = autocommit
self.session.autoflush = autoflush