def get(self, attr=None, exclude_attr=None):
"""Get detailed information about the instance and related components"""
gets = {
'annotations': lambda: DeisCkanInstanceAnnotations(self).get(),
'db': lambda: DeisCkanInstanceDb(self, 'db').get(),
'datastore': lambda: DeisCkanInstanceDb(self, 'datastore').get(),
'deployment': lambda: DeisCkanInstanceDeployment(self).get(),
'envvars': lambda: DeisCkanInstanceEnvvars(self).get(),
'namespace': lambda: DeisCkanInstanceNamespace(self).get(),
'registry': lambda: DeisCkanInstanceRegistry(self).get(),
'solr': lambda: DeisCkanInstanceSolr(self).get(),
'storage': lambda: DeisCkanInstanceStorage(self).get(),
}
if exclude_attr:
gets = {k: v for k, v in gets.items() if k not in exclude_attr}
if attr:
return gets[attr]()
else:
ret = {'ready': True}
for k, v in gets.items():
ret[k] = v()
if type(ret[k]) == dict and not ret[k].get('ready'):
ret['ready'] = False
ret['id'] = self.id
return ret
def update(self, wait_ready=False, skip_solr=False, skip_deployment=False):
"""Ensure the instance is updated to latest spec"""
old_deployment = kubectl.get(f'deployment {self.id}', required=False, namespace=self.id)
if old_deployment:
old_deployment_generation = old_deployment.get('metadata', {}).get('generation')
else:
old_deployment_generation = None
if old_deployment_generation:
expected_new_deployment_generation = old_deployment_generation + 1
else:
expected_new_deployment_generation = 1
print(f'old deployment generation = {old_deployment_generation}')
DeisCkanInstanceNamespace(self).update()
DeisCkanInstanceDb(self, 'db').update()
DeisCkanInstanceDb(self, 'datastore').update()
if not skip_solr:
DeisCkanInstanceSolr(self).update()
DeisCkanInstanceStorage(self).update()
DeisCkanInstanceRegistry(self).update()
envvars = DeisCkanInstanceEnvvars(self)
envvars.update()
if not skip_deployment:
DeisCkanInstanceDeployment(self).update()
while True:
time.sleep(.2)
new_deployment = kubectl.get(f'deployment {self.id}', required=False, namespace=self.id)
if not new_deployment: continue
new_deployment_generation = new_deployment.get('metadata', {}).get('generation')
if not new_deployment_generation: continue
if new_deployment_generation == old_deployment_generation: continue
if new_deployment_generation != expected_new_deployment_generation:
raise Exception(f'Invalid generation: {new_deployment_generation} '
f'(expected: {expected_new_deployment_generation}')
print(f'new deployment generation: {new_deployment_generation}')
break
if wait_ready:
print('Waiting for ready status')
time.sleep(3)
while True:
data = self.get()
if data.get('ready'):
print(yaml.dump(data, default_flow_style=False))
break
else:
print(yaml.dump(
{
k: v for k, v in data.items()
if (k not in ['ready'] and type(v) == dict and not v.get('ready')) or k == 'namespace'
},
default_flow_style=False)
)
time.sleep(2)
self.ckan.update()
try:
DeisCkanInstanceDb(self, 'datastore').set_datastore_readonly_permissions()
except Exception:
logs.warning('Setting datastore permissions failed, continuing anyway')
# Create/Update uptime monitoring after everything else is ready
DeisCkanInstanceUptime(self).update(envvars.site_url)
class DbTestCase(unittest.TestCase):
def setUp(self):
self.instance = DeisCkanInstance('montreal')
self.instance._spec = Mock(datastore={'name': 'test'})
self.db = DeisCkanInstanceDb(self.instance, 'db')
self.datastore = DeisCkanInstanceDb(self.instance, 'datastore')
def test_set_datastore_readonly_permissions(self):
postgres_driver.connect = MagicMock()
db_manager.get_external_admin_connection_string = MagicMock()
conn = MagicMock()
cursor = MagicMock()
conn.cursor.return_value.__enter__.return_value = cursor
postgres_driver.connect.return_value.__enter__.return_value = conn
self.instance.annotations.get_secret = lambda x: 'postgres_ro'
self.datastore.set_datastore_readonly_permissions()
self.assertEqual(cursor.execute.call_count, 5)
cursor.execute.assert_has_calls([
call("GRANT CONNECT ON DATABASE \"test\" TO \"postgres_ro\";"),
call("GRANT USAGE ON SCHEMA public TO \"postgres_ro\";"),
call(
"GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"postgres_ro\";"
),
call(
"ALTER DEFAULT PRIVILEGES FOR USER \"test\" IN SCHEMA public GRANT SELECT ON TABLES TO \"postgres_ro\";"
),
call(
DATASTORE_PERMISSIONS_SQL_TEMPLATE.replace(
'{{SITE_USER}}', 'test').replace('{{DS_RO_USER}}',
'postgres_ro'))
])
def test_set_datastore_readonly_permissions_for_db(self):
with self.assertRaises(AssertionError):
self.db.set_datastore_readonly_permissions()
def setUp(self):
self.instance = DeisCkanInstance('montreal')
self.instance._spec = Mock(datastore={'name': 'test'})
self.db = DeisCkanInstanceDb(self.instance, 'db')
self.datastore = DeisCkanInstanceDb(self.instance, 'datastore')