def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_LW('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_LE('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(
_LW('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_LE('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
def test_update_environment(self):
"""Check that environment rename works."""
self._set_policy_rules(
{'show_environment': '@',
'update_environment': '@'}
)
self.expect_policy_check('update_environment',
{'environment_id': '12345'})
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
expected = dict(
id='12345',
name='my-env',
version=0,
networking={},
created=fake_now,
updated=fake_now,
tenant_id=self.tenant,
description={
'Objects': {
'?': {'id': '12345'}
},
'Attributes': {}
}
)
e = models.Environment(**expected)
test_utils.save_models(e)
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
del expected['description']
expected['services'] = []
expected['status'] = 'ready'
expected['name'] = 'renamed env'
expected['updated'] = fake_now
body = {
'name': 'renamed env'
}
req = self._put('/environments/12345', json.dumps(body))
result = req.get_response(self.api)
self.assertEqual(200, result.status_code)
self.expect_policy_check('show_environment',
{'environment_id': '12345'})
req = self._get('/environments/12345')
result = req.get_response(self.api)
self.assertEqual(200, result.status_code)
expected['created'] = timeutils.isotime(expected['created'])[:-1]
expected['updated'] = timeutils.isotime(expected['updated'])[:-1]
self.assertEqual(expected, json.loads(result.body))
def test_update_environment(self):
"""Check that environment rename works."""
self._set_policy_rules({
'show_environment': '@',
'update_environment': '@'
})
self.expect_policy_check('update_environment',
{'environment_id': '12345'})
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
expected = dict(id='12345',
name='my-env',
version=0,
networking={},
created=fake_now,
updated=fake_now,
tenant_id=self.tenant,
description={
'Objects': {
'?': {
'id': '12345'
}
},
'Attributes': []
})
e = models.Environment(**expected)
test_utils.save_models(e)
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
del expected['description']
expected['services'] = []
expected['status'] = 'ready'
expected['name'] = 'renamed_env'
expected['updated'] = fake_now
body = {'name': 'renamed_env'}
req = self._put('/environments/12345', json.dumps(body))
result = req.get_response(self.api)
self.assertEqual(200, result.status_code)
self.expect_policy_check('show_environment',
{'environment_id': '12345'})
req = self._get('/environments/12345')
result = req.get_response(self.api)
self.assertEqual(200, result.status_code)
expected['created'] = timeutils.isotime(expected['created'])[:-1]
expected['updated'] = timeutils.isotime(expected['updated'])[:-1]
self.assertEqual(expected, json.loads(result.body))
def test_delete_environment(self):
"""Test that environment deletion results in the correct rpc call."""
self._set_policy_rules({'delete_environment': '@'})
self.expect_policy_check('delete_environment',
{'environment_id': '12345'})
fake_now = timeutils.utcnow()
expected = dict(id='12345',
name='my-env',
version=0,
networking={},
created=fake_now,
updated=fake_now,
tenant_id=self.tenant,
description={
'Objects': {
'?': {
'id': '12345'
}
},
'Attributes': {}
})
e = models.Environment(**expected)
test_utils.save_models(e)
req = self._delete('/environments/12345')
result = req.get_response(self.api)
# Should this be expected behavior?
self.assertEqual('', result.body)
self.assertEqual(200, result.status_code)
def test_delete_environment(self):
"""Test that environment deletion results in the correct rpc call."""
self._set_policy_rules(
{'delete_environment': '@'}
)
self.expect_policy_check(
'delete_environment', {'environment_id': '12345'}
)
fake_now = timeutils.utcnow()
expected = dict(
id='12345',
name='my-env',
version=0,
networking={},
created=fake_now,
updated=fake_now,
tenant_id=self.tenant,
description={
'Objects': {
'?': {'id': '12345'}
},
'Attributes': {}
}
)
e = models.Environment(**expected)
test_utils.save_models(e)
req = self._delete('/environments/12345')
result = req.get_response(self.api)
# Should this be expected behavior?
self.assertEqual('', result.body)
self.assertEqual(200, result.status_code)
def test_update_environment(self, mock_policy_check):
"""Check that environment rename works"""
self._mock_policy_setup(mock_policy_check, 'update_environment',
target={'environment_id': '12345'})
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
expected = dict(
id='12345',
name='my-env',
version=0,
networking={},
created=fake_now,
updated=fake_now,
tenant_id=self.tenant,
description={
'Objects': {
'?': {'id': '12345'}
},
'Attributes': {}
}
)
e = models.Environment(**expected)
test_utils.save_models(e)
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
del expected['description']
expected['services'] = []
expected['status'] = 'ready'
expected['name'] = 'renamed env'
expected['updated'] = fake_now
body = {
'name': 'renamed env'
}
req = self._post('/environments/12345', json.dumps(body))
result = self.controller.update(req, '12345', body)
self._mock_policy_setup(mock_policy_check, 'show_environment',
target={'environment_id': '12345'})
req = self._get('/environments/12345')
result = self.controller.show(req, '12345')
self.assertEqual(expected, result)
def process_result(context, result):
secure_result = token_sanitizer.TokenSanitizer().sanitize(result)
LOG.debug('Got result from orchestration '
'engine:\n{0}'.format(secure_result))
if not result['Objects']:
LOG.debug('Ignoring result for deleted environment')
return
result_id = result['Objects']['?']['id']
unit = session.get_session()
environment = unit.query(models.Environment).get(result_id)
if not environment:
LOG.warning(_('Environment result could not be handled, specified '
'environment was not found in database'))
return
environment.description = result
environment.description['Objects']['services'] = \
environment.description['Objects'].get('applications', [])
del environment.description['Objects']['applications']
environment.networking = result.get('networking', {})
environment.version += 1
environment.save(unit)
#close session
conf_session = unit.query(models.Session).filter_by(
**{'environment_id': environment.id, 'state': 'deploying'}).first()
conf_session.state = 'deployed'
conf_session.save(unit)
#close deployment
deployment = get_last_deployment(unit, environment.id)
deployment.finished = timeutils.utcnow()
num_errors = unit.query(models.Status)\
.filter_by(level='error', deployment_id=deployment.id).count()
num_warnings = unit.query(models.Status)\
.filter_by(level='warning', deployment_id=deployment.id).count()
final_status_text = "Deployment finished"
if num_errors:
final_status_text += " with errors"
elif num_warnings:
final_status_text += " with warnings"
status = models.Status()
status.deployment_id = deployment.id
status.text = final_status_text
status.level = 'info'
deployment.statuses.append(status)
deployment.save(unit)
def test_create_environment(self):
"""Create an environment, test environment.show()."""
self._set_policy_rules({
'list_environments': '@',
'create_environment': '@',
'show_environment': '@'
})
self.expect_policy_check('create_environment')
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
uuids = ('env_object_id', 'network_id', 'environment_id')
mock_uuid = self._stub_uuid(uuids)
expected = {
'tenant_id': self.tenant,
'id': 'environment_id',
'name': 'my_env',
'networking': {},
'version': 0,
# TODO(sjmc7) - bug 1347298
'created': timeutils.isotime(fake_now)[:-1],
'updated': timeutils.isotime(fake_now)[:-1]
}
body = {'name': 'my_env'}
req = self._post('/environments', json.dumps(body))
result = req.get_response(self.api)
self.assertEqual(expected, json.loads(result.body))
expected['status'] = 'ready'
# Reset the policy expectation
self.expect_policy_check('list_environments')
req = self._get('/environments')
result = req.get_response(self.api)
self.assertEqual(200, result.status_code)
self.assertEqual({'environments': [expected]}, json.loads(result.body))
expected['services'] = []
# Reset the policy expectation
self.expect_policy_check('show_environment',
{'environment_id': uuids[-1]})
req = self._get('/environments/%s' % uuids[-1])
result = req.get_response(self.api)
self.assertEqual(expected, json.loads(result.body))
self.assertEqual(3, mock_uuid.call_count)
def put_data(environment_id, session_id, data, path):
get_description = envs.EnvironmentServices.get_environment_description
save_description = envs.EnvironmentServices.\
save_environment_description
env_description = get_description(environment_id, session_id)
utils.TraverseHelper.update(path, data, env_description)
env_description['?']['updated'] = str(timeutils.utcnow())
save_description(session_id, env_description)
return data
def put_data(environment_id, session_id, data, path):
get_description = envs.EnvironmentServices.get_environment_description
save_description = envs.EnvironmentServices.\
save_environment_description
env_description = get_description(environment_id, session_id)
utils.TraverseHelper.update(path, data, env_description)
env_description['?']['updated'] = str(timeutils.utcnow())
save_description(session_id, env_description)
return data
def drop_old_duplicate_entries_from_table(migrate_engine, table_name,
use_soft_delete, *uc_column_names):
"""Drop all old rows having the same values for columns in uc_columns.
This method drop (or mark ad `deleted` if use_soft_delete is True) old
duplicate rows form table with name `table_name`.
:param migrate_engine: Sqlalchemy engine
:param table_name: Table with duplicates
:param use_soft_delete: If True - values will be marked as `deleted`,
if False - values will be removed from table
:param uc_column_names: Unique constraint columns
"""
meta = MetaData()
meta.bind = migrate_engine
table = Table(table_name, meta, autoload=True)
columns_for_group_by = [table.c[name] for name in uc_column_names]
columns_for_select = [func.max(table.c.id)]
columns_for_select.extend(columns_for_group_by)
duplicated_rows_select = sqlalchemy.sql.select(
columns_for_select, group_by=columns_for_group_by,
having=func.count(table.c.id) > 1)
for row in migrate_engine.execute(duplicated_rows_select):
# NOTE(boris-42): Do not remove row that has the biggest ID.
delete_condition = table.c.id != row[0]
is_none = None # workaround for pyflakes
delete_condition &= table.c.deleted_at == is_none
for name in uc_column_names:
delete_condition &= table.c[name] == row[name]
rows_to_delete_select = sqlalchemy.sql.select(
[table.c.id]).where(delete_condition)
for row in migrate_engine.execute(rows_to_delete_select).fetchall():
LOG.info(_LI("Deleting duplicated row with id: %(id)s from table: "
"%(table)s") % dict(id=row[0], table=table_name))
if use_soft_delete:
delete_statement = table.update().\
where(delete_condition).\
values({
'deleted': literal_column('id'),
'updated_at': literal_column('updated_at'),
'deleted_at': timeutils.utcnow()
})
else:
delete_statement = table.delete().where(delete_condition)
migrate_engine.execute(delete_statement)
def test_create_environment(self):
"""Create an environment, test environment.show()."""
self._set_policy_rules(
{'list_environments': '@',
'create_environment': '@',
'show_environment': '@'}
)
self.expect_policy_check('create_environment')
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
uuids = ('env_object_id', 'network_id', 'environment_id')
mock_uuid = self._stub_uuid(uuids)
expected = {'tenant_id': self.tenant,
'id': 'environment_id',
'name': 'my_env',
'networking': {},
'version': 0,
# TODO(sjmc7) - bug 1347298
'created': timeutils.isotime(fake_now)[:-1],
'updated': timeutils.isotime(fake_now)[:-1]}
body = {'name': 'my_env'}
req = self._post('/environments', json.dumps(body))
result = req.get_response(self.api)
self.assertEqual(expected, json.loads(result.body))
expected['status'] = 'ready'
# Reset the policy expectation
self.expect_policy_check('list_environments')
req = self._get('/environments')
result = req.get_response(self.api)
self.assertEqual(200, result.status_code)
self.assertEqual({'environments': [expected]}, json.loads(result.body))
expected['services'] = []
# Reset the policy expectation
self.expect_policy_check('show_environment',
{'environment_id': uuids[-1]})
req = self._get('/environments/%s' % uuids[-1])
result = req.get_response(self.api)
self.assertEqual(expected, json.loads(result.body))
self.assertEqual(3, mock_uuid.call_count)
def _create_default_categories(op):
bind = op.get_bind()
table = sa_table(
'category',
sa.Column('id', sa.String(length=36), primary_key=True),
sa.Column('created', sa.DateTime()),
sa.Column('updated', sa.DateTime()),
sa.Column('name', sa.String(length=80)))
now = timeutils.utcnow()
for category in consts.CATEGORIES:
values = {'id': uuid.uuid4().hex,
'name': category,
'updated': now,
'created': now}
bind.execute(table.insert(values=values))
def _create_default_categories(op):
bind = op.get_bind()
table = sa_table('category',
sa.Column('id', sa.String(length=36), primary_key=True),
sa.Column('created', sa.DateTime()),
sa.Column('updated', sa.DateTime()),
sa.Column('name', sa.String(length=80)))
now = timeutils.utcnow()
for category in consts.CATEGORIES:
values = {
'id': uuid.uuid4().hex,
'name': category,
'updated': now,
'created': now
}
bind.execute(table.insert(values=values))
def test_create_environment(self, mock_policy_check):
"""Create an environment, test environment.show()"""
self._mock_policy_setup(mock_policy_check, 'create_environment')
fake_now = timeutils.utcnow()
timeutils.utcnow.override_time = fake_now
uuids = ('env_object_id', 'network_id', 'environment_id')
mock_uuid = self._stub_uuid(uuids)
expected = {'tenant_id': self.tenant,
'id': 'environment_id',
'name': 'my_env',
'networking': {},
'version': 0,
'created': fake_now,
'updated': fake_now}
body = {'name': 'my_env'}
req = self._post('/environments', json.dumps(body))
result = self.controller.create(req, body)
self.assertEqual(expected, result)
expected['status'] = 'ready'
# Reset the policy expectation
self._mock_policy_setup(mock_policy_check, 'list_environments')
req = self._get('/environments')
result = self.controller.index(req)
self.assertEqual({'environments': [expected]}, result)
expected['services'] = []
# Reset the policy expectation
self._mock_policy_setup(mock_policy_check, 'show_environment',
target={'environment_id': uuids[-1]})
req = self._get('/environments/%s' % uuids[-1])
result = self.controller.show(req, uuids[-1])
self.assertEqual(expected, result)
self.assertEqual(3, mock_uuid.call_count)
def test_delete_environment(self, mock_policy_check):
"""Test that environment deletion results in the correct rpc call"""
self._mock_policy_setup(mock_policy_check, 'delete_environment',
target={'environment_id': '12345'})
fake_now = timeutils.utcnow()
expected = dict(
id='12345',
name='my-env',
version=0,
networking={},
created=fake_now,
updated=fake_now,
tenant_id=self.tenant,
description={
'Objects': {
'?': {'id': '12345'}
},
'Attributes': {}
}
)
e = models.Environment(**expected)
test_utils.save_models(e)
rpc_task = {
'tenant_id': self.tenant,
'model': {'Attributes': {}, 'Objects': None},
'token': None
}
req = self._delete('/environments/12345')
result = self.controller.delete(req, '12345')
self.mock_engine_rpc.handle_task.assert_called_once_with(rpc_task)
# Should this be expected behavior?
self.assertEqual(None, result)
def update(self, values):
"""dict.update() behaviour."""
self.updated = timeutils.utcnow()
super(_MuranoBase, self).update(values)
def soft_delete(self, synchronize_session='evaluate'):
return self.update({'deleted': literal_column('id'),
'updated_at': literal_column('updated_at'),
'deleted_at': timeutils.utcnow()},
synchronize_session=synchronize_session)
def test_execute_action(self, mock_policy_check):
"""Test that action execution results in the correct rpc call."""
self._set_policy_rules({'execute_action': '@'})
fake_now = timeutils.utcnow()
expected = dict(id='12345',
name='my-env',
version=0,
networking={},
created=fake_now,
updated=fake_now,
tenant_id=self.tenant,
description={
'Objects': {
'?': {
'id': '12345',
'_actions': {
'actionsID_action': {
'enabled': True,
'name': 'Testaction'
}
}
}
},
'Attributes': {}
})
e = models.Environment(**expected)
test_utils.save_models(e)
rpc_task = {
'tenant_id': self.tenant,
'model': {
'Objects': {
'applications': [],
'?': {
'_actions': {
'actionsID_action': {
'name': 'Testaction',
'enabled': True
}
},
'id': '12345'
}
},
'Attributes': {}
},
'action': {
'method': 'Testaction',
'object_id': '12345',
'args': '{}'
},
'token': None,
'id': '12345'
}
req = self._post('/environments/12345/actions/actionID_action', '{}')
result = self.controller.execute(req, '12345', 'actionsID_action',
'{}')
self.mock_engine_rpc.handle_task.assert_called_once_with(rpc_task)
# Should this be expected behavior?
self.assertEqual(None, result)
def __setitem__(self, key, value):
self.updated = timeutils.utcnow()
super(_MuranoBase, self).__setitem__(key, value)
def update(self, values):
"""dict.update() behaviour."""
self.updated = timeutils.utcnow()
super(_MuranoBase, self).update(values)
def __setitem__(self, key, value):
self.updated = timeutils.utcnow()
super(ModificationsTrackedObject, self).__setitem__(key, value)
def update(self, values):
"""dict.update() behaviour."""
self.updated = timeutils.utcnow()
super(ModificationsTrackedObject, self).update(values)
def process_result(context, result, environment_id):
secure_result = token_sanitizer.TokenSanitizer().sanitize(result)
LOG.debug('Got result from orchestration '
'engine:\n{0}'.format(secure_result))
unit = session.get_session()
environment = unit.query(models.Environment).get(environment_id)
if not environment:
LOG.warning(_('Environment result could not be handled, specified '
'environment was not found in database'))
return
if result['Objects'] is None and result.get('ObjectsCopy', {}) is None:
environments.EnvironmentServices.remove(environment_id)
return
environment.description = result
if environment.description['Objects'] is not None:
environment.description['Objects']['services'] = \
environment.description['Objects'].pop('applications', [])
# environment.networking = result.get('networking', {})
action_name = 'Deployment'
deleted = False
else:
action_name = 'Deletion'
deleted = True
environment.version += 1
environment.save(unit)
#close deployment
deployment = get_last_deployment(unit, environment.id)
deployment.finished = timeutils.utcnow()
num_errors = unit.query(models.Status)\
.filter_by(level='error', task_id=deployment.id).count()
num_warnings = unit.query(models.Status)\
.filter_by(level='warning', task_id=deployment.id).count()
final_status_text = action_name + ' finished'
if num_errors:
final_status_text += " with errors"
elif num_warnings:
final_status_text += " with warnings"
status = models.Status()
status.task_id = deployment.id
status.text = final_status_text
status.level = 'info'
deployment.statuses.append(status)
deployment.save(unit)
#close session
conf_session = unit.query(models.Session).filter_by(
**{'environment_id': environment.id,
'state': 'deploying' if not deleted else 'deleting'}).first()
if num_errors > 0:
conf_session.state = \
sessions.SessionState.DELETE_FAILURE if deleted else \
sessions.SessionState.DEPLOY_FAILURE
else:
conf_session.state = sessions.SessionState.DEPLOYED
conf_session.save(unit)
def soft_delete(self, session):
"""Mark this object as deleted."""
self.deleted = self.id
self.deleted_at = timeutils.utcnow()
self.save(session=session)
def process_result(context, result, environment_id):
secure_result = token_sanitizer.TokenSanitizer().sanitize(result)
LOG.debug('Got result from orchestration '
'engine:\n{0}'.format(secure_result))
unit = session.get_session()
environment = unit.query(models.Environment).get(environment_id)
if not environment:
LOG.warning(
_('Environment result could not be handled, specified '
'environment was not found in database'))
return
if result['Objects'] is None and result.get('ObjectsCopy', {}) is None:
environments.EnvironmentServices.remove(environment_id)
return
environment.description = result
if environment.description['Objects'] is not None:
environment.description['Objects']['services'] = \
environment.description['Objects'].pop('applications', [])
# environment.networking = result.get('networking', {})
action_name = 'Deployment'
deleted = False
else:
action_name = 'Deletion'
deleted = True
environment.version += 1
environment.save(unit)
#close deployment
deployment = get_last_deployment(unit, environment.id)
deployment.finished = timeutils.utcnow()
num_errors = unit.query(models.Status)\
.filter_by(level='error', task_id=deployment.id).count()
num_warnings = unit.query(models.Status)\
.filter_by(level='warning', task_id=deployment.id).count()
final_status_text = action_name + ' finished'
if num_errors:
final_status_text += " with errors"
elif num_warnings:
final_status_text += " with warnings"
status = models.Status()
status.task_id = deployment.id
status.text = final_status_text
status.level = 'info'
deployment.statuses.append(status)
deployment.save(unit)
#close session
conf_session = unit.query(models.Session).filter_by(
**{
'environment_id': environment.id,
'state': 'deploying' if not deleted else 'deleting'
}).first()
if num_errors > 0:
conf_session.state = \
sessions.SessionState.DELETE_FAILURE if deleted else \
sessions.SessionState.DEPLOY_FAILURE
else:
conf_session.state = sessions.SessionState.DEPLOYED
conf_session.save(unit)
def __setitem__(self, key, value):
self.updated = timeutils.utcnow()
super(_MuranoBase, self).__setitem__(key, value)
