def quota_destroy_all_by_project(context, project_id):
session = get_session()
with session.begin():
model_query(context, models.Quota, session=session,
read_deleted="no").\
filter_by(project_id=project_id).\
update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')},
synchronize_session=False)
model_query(context, models.ProjectUserQuota, session=session,
read_deleted="no").\
filter_by(project_id=project_id).\
update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')},
synchronize_session=False)
model_query(context, models.QuotaUsage,
session=session, read_deleted="no").\
filter_by(project_id=project_id).\
update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')},
synchronize_session=False)
model_query(context, models.Reservation,
session=session, read_deleted="no").\
filter_by(project_id=project_id).\
update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')},
synchronize_session=False)
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_LW('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_LE('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
def test_share_host_update_db(self):
self.mox.StubOutWithMock(timeutils, 'utcnow')
self.mox.StubOutWithMock(db, 'share_update')
timeutils.utcnow().AndReturn('fake-now')
db.share_update(self.context, 31337,
{'host': 'fake_host',
'scheduled_at': 'fake-now'})
self.mox.ReplayAll()
driver.share_update_db(self.context, 31337, 'fake_host')
def inner(self, *args, **kwargs):
LOG.debug("Entering %(cls)s.%(method)s",
{'cls': self.__class__.__name__,
'method': func.__name__})
start = timeutils.utcnow()
ret = func(self, *args, **kwargs)
end = timeutils.utcnow()
LOG.debug("Exiting %(cls)s.%(method)s. "
"Spent %(duration)s sec. "
"Return %(return)s",
{'cls': self.__class__.__name__,
'duration': timeutils.delta_seconds(start, end),
'method': func.__name__,
'return': ret})
return ret
def reservation_expire(context):
session = get_session()
with session.begin():
current_time = timeutils.utcnow()
reservation_query = model_query(context, models.Reservation,
session=session, read_deleted="no").\
filter(models.Reservation.expire < current_time)
for reservation in reservation_query.join(models.QuotaUsage).all():
if reservation.delta >= 0:
reservation.usage.reserved -= reservation.delta
session.add(reservation.usage)
reservation_query.update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')})
def index(self, req):
"""Return a list of all running services. """
context = req.environ['manila.context']
authorize(context)
now = timeutils.utcnow()
all_services = db.service_get_all(context)
services = []
for service in all_services:
service = {
'binary': service['binary'],
'host': service['host'],
'zone': service['availability_zone'],
'status': 'disabled' if service['disabled'] else 'enabled',
'state': 'up' if utils.service_is_up(service) else 'down',
'updated_at': service['updated_at'],
}
services.append(service)
search_opts = [
'host',
'binary',
'zone',
'state',
'status',
]
for search_opt in search_opts:
value = ''
if search_opt in req.GET:
value = req.GET[search_opt]
services = [s for s in services if s[search_opt] == value]
if len(services) == 0:
break
return {'services': services}
def create_share(self, context, share_id, request_spec=None,
filter_properties=None, snapshot_id=None):
"""Creates a share."""
context = context.elevated()
if filter_properties is None:
filter_properties = {}
share_ref = self.db.share_get(context, share_id)
if snapshot_id is not None:
snapshot_ref = self.db.share_snapshot_get(context, snapshot_id)
else:
snapshot_ref = None
try:
if snapshot_ref:
self.driver.allocate_container_from_snapshot(context,
share_ref,
snapshot_ref)
else:
self.driver.allocate_container(context, share_ref)
export_location = self.driver.create_share(context, share_ref)
self.db.share_update(context, share_id,
{'export_location': export_location})
self.driver.create_export(context, share_ref)
except Exception:
with excutils.save_and_reraise_exception():
self.db.share_update(context, share_id, {'status': 'error'})
else:
self.db.share_update(context, share_id,
{'status': 'available',
'launched_at': timeutils.utcnow()})
def __init__(self,
user_id,
project_id,
is_admin=None,
read_deleted="no",
roles=None,
remote_address=None,
timestamp=None,
request_id=None,
auth_token=None,
overwrite=True,
quota_class=None,
service_catalog=None,
**kwargs):
"""
:param read_deleted: 'no' indicates deleted records are hidden, 'yes'
indicates deleted records are visible, 'only' indicates that
*only* deleted records are visible.
:param overwrite: Set to False to ensure that the greenthread local
copy of the index is not overwritten.
:param kwargs: Extra arguments that might be present, but we ignore
because they possibly came in from older rpc messages.
"""
if kwargs:
LOG.warn(
_('Arguments dropped when creating context: %s') % str(kwargs))
self.user_id = user_id
self.project_id = project_id
self.roles = roles or []
self.is_admin = is_admin
if self.is_admin is None:
self.is_admin = policy.check_is_admin(self.roles)
elif self.is_admin and 'admin' not in self.roles:
self.roles.append('admin')
self.read_deleted = read_deleted
self.remote_address = remote_address
if not timestamp:
timestamp = timeutils.utcnow()
if isinstance(timestamp, six.string_types):
timestamp = timeutils.parse_strtime(timestamp)
self.timestamp = timestamp
if service_catalog:
self.service_catalog = [
s for s in service_catalog
if s.get('type') in ('compute', 'volume')
]
else:
self.service_catalog = []
if not request_id:
request_id = generate_request_id()
self.request_id = request_id
self.auth_token = auth_token
self.quota_class = quota_class
if overwrite or not hasattr(local.store, 'context'):
self.update_store()
def share_update_db(context, share_id, host):
'''Set the host and set the scheduled_at field of a share.
:returns: A Share with the updated fields set properly.
'''
now = timeutils.utcnow()
values = {'host': host, 'scheduled_at': now}
return db.share_update(context, share_id, values)
def share_delete(context, share_id):
session = get_session()
share_ref = share_get(context, share_id, session)
share_ref.update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at'),
'status': 'deleted'})
share_ref.save(session)
def share_update_db(context, share_id, host):
'''Set the host and set the scheduled_at field of a share.
:returns: A Share with the updated fields set properly.
'''
now = timeutils.utcnow()
values = {'host': host, 'scheduled_at': now}
return db.share_update(context, share_id, values)
def share_access_delete(context, access_id):
session = get_session()
with session.begin():
session.query(models.ShareAccessMapping).\
filter_by(id=access_id).\
update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at'),
'state': models.ShareAccessMapping.STATE_DELETED})
def share_snapshot_destroy(context, snapshot_id):
session = get_session()
with session.begin():
session.query(models.ShareSnapshot).\
filter_by(id=snapshot_id).\
update({'status': 'deleted',
'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')})
def inner(self, *args, **kwargs):
LOG.debug("Entering %(cls)s.%(method)s", {
'cls': self.__class__.__name__,
'method': func.__name__
})
start = timeutils.utcnow()
ret = func(self, *args, **kwargs)
end = timeutils.utcnow()
LOG.debug(
"Exiting %(cls)s.%(method)s. "
"Spent %(duration)s sec. "
"Return %(return)s", {
'cls': self.__class__.__name__,
'duration': timeutils.delta_seconds(start, end),
'method': func.__name__,
'return': ret
})
return ret
def mock_host_manager_db_calls(mock_obj, disabled=None):
services = [
dict(id=1, host='host1', topic='share', disabled=False,
availability_zone='zone1', updated_at=timeutils.utcnow()),
dict(id=2, host='host2', topic='share', disabled=False,
availability_zone='zone1', updated_at=timeutils.utcnow()),
dict(id=3, host='host3', topic='share', disabled=False,
availability_zone='zone2', updated_at=timeutils.utcnow()),
dict(id=4, host='host4', topic='share', disabled=False,
availability_zone='zone3', updated_at=timeutils.utcnow()),
# service on host5 is disabled
dict(id=5, host='host5', topic='share', disabled=True,
availability_zone='zone4', updated_at=timeutils.utcnow()),
]
if disabled is None:
mock_obj.return_value = services
else:
mock_obj.return_value = [service for service in services
if service['disabled'] == disabled]
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone, e:
self.stop()
done.send(e.retvalue)
def notify(context, publisher_id, event_type, priority, payload):
"""Sends a notification using the specified driver
:param publisher_id: the source worker_type.host of the message
:param event_type: the literal type of event (ex. Instance Creation)
:param priority: patterned after the enumeration of Python logging
levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL)
:param payload: A python dictionary of attributes
Outgoing message format includes the above parameters, and appends the
following:
message_id
a UUID representing the id for this notification
timestamp
the GMT timestamp the notification was sent at
The composite message will be constructed as a dictionary of the above
attributes, which will then be sent via the transport mechanism defined
by the driver.
Message example::
{'message_id': str(uuid.uuid4()),
'publisher_id': 'compute.host1',
'timestamp': timeutils.utcnow(),
'priority': 'WARN',
'event_type': 'compute.create_instance',
'payload': {'instance_id': 12, ... }}
"""
if priority not in log_levels:
raise BadPriorityException(_("%s not in valid priorities") % priority)
# Ensure everything is JSON serializable.
payload = jsonutils.to_primitive(payload, convert_instances=True)
msg = dict(
message_id=str(uuid.uuid4()),
publisher_id=publisher_id,
event_type=event_type,
priority=priority,
payload=payload,
timestamp=str(timeutils.utcnow()),
)
for driver in _get_drivers():
try:
driver.notify(context, msg)
except Exception as e:
LOG.exception(
_LE("Problem '%(e)s' attempting to " "send to notification system. " "Payload=%(payload)s")
% dict(e=e, payload=payload)
)
def consume_from_share(self, share):
"""Incrementally update host state from an share"""
share_gb = share['size']
if self.free_capacity_gb == 'infinite':
# There's virtually infinite space on back-end
pass
elif self.free_capacity_gb == 'unknown':
# Unable to determine the actual free space on back-end
pass
else:
self.free_capacity_gb -= share_gb
self.updated = timeutils.utcnow()
def consume_from_share(self, share):
"""Incrementally update host state from an share"""
share_gb = share['size']
if self.free_capacity_gb == 'infinite':
# There's virtually infinite space on back-end
pass
elif self.free_capacity_gb == 'unknown':
# Unable to determine the actual free space on back-end
pass
else:
self.free_capacity_gb -= share_gb
self.updated = timeutils.utcnow()
def test_update_service_capabilities_for_shares(self):
service_states = self.host_manager.service_states
self.assertDictMatch(service_states, {})
self.mox.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().AndReturn(31337)
timeutils.utcnow().AndReturn(31338)
timeutils.utcnow().AndReturn(31339)
host1_share_capabs = dict(free_capacity_gb=4321, timestamp=1)
host2_share_capabs = dict(free_capacity_gb=5432, timestamp=1)
host3_share_capabs = dict(free_capacity_gb=6543, timestamp=1)
self.mox.ReplayAll()
service_name = 'share'
self.host_manager.update_service_capabilities(service_name, 'host1',
host1_share_capabs)
self.host_manager.update_service_capabilities(service_name, 'host2',
host2_share_capabs)
self.host_manager.update_service_capabilities(service_name, 'host3',
host3_share_capabs)
# Make sure dictionary isn't re-assigned
self.assertEqual(self.host_manager.service_states, service_states)
# Make sure original dictionary wasn't copied
self.assertEqual(host1_share_capabs['timestamp'], 1)
host1_share_capabs['timestamp'] = 31337
host2_share_capabs['timestamp'] = 31338
host3_share_capabs['timestamp'] = 31339
expected = {'host1': host1_share_capabs,
'host2': host2_share_capabs,
'host3': host3_share_capabs}
self.assertDictMatch(service_states, expected)
def mock_host_manager_db_calls(mock_obj, disabled=None):
services = [
dict(id=1,
host='host1',
topic='share',
disabled=False,
availability_zone='zone1',
updated_at=timeutils.utcnow()),
dict(id=2,
host='host2',
topic='share',
disabled=False,
availability_zone='zone1',
updated_at=timeutils.utcnow()),
dict(id=3,
host='host3',
topic='share',
disabled=False,
availability_zone='zone2',
updated_at=timeutils.utcnow()),
dict(id=4,
host='host4',
topic='share',
disabled=False,
availability_zone='zone3',
updated_at=timeutils.utcnow()),
# service on host5 is disabled
dict(id=5,
host='host5',
topic='share',
disabled=True,
availability_zone='zone4',
updated_at=timeutils.utcnow()),
]
if disabled is None:
mock_obj.return_value = services
else:
mock_obj.return_value = [
service for service in services if service['disabled'] == disabled
]
def test_service_is_up(self):
fts_func = datetime.datetime.fromtimestamp
fake_now = 1000
down_time = 5
self.flags(service_down_time=down_time)
self.mox.StubOutWithMock(timeutils, 'utcnow')
# Up (equal)
timeutils.utcnow().AndReturn(fts_func(fake_now))
service = {'updated_at': fts_func(fake_now - down_time),
'created_at': fts_func(fake_now - down_time)}
self.mox.ReplayAll()
result = utils.service_is_up(service)
self.assertTrue(result)
self.mox.ResetAll()
# Up
timeutils.utcnow().AndReturn(fts_func(fake_now))
service = {'updated_at': fts_func(fake_now - down_time + 1),
'created_at': fts_func(fake_now - down_time + 1)}
self.mox.ReplayAll()
result = utils.service_is_up(service)
self.assertTrue(result)
self.mox.ResetAll()
# Down
timeutils.utcnow().AndReturn(fts_func(fake_now))
service = {'updated_at': fts_func(fake_now - down_time - 1),
'created_at': fts_func(fake_now - down_time - 1)}
self.mox.ReplayAll()
result = utils.service_is_up(service)
self.assertFalse(result)
def reservation_rollback(context, reservations, project_id=None, user_id=None):
session = get_session()
with session.begin():
usages = _get_user_quota_usages(context, session, project_id, user_id)
reservation_query = _quota_reservations_query(session, context,
reservations)
for reservation in reservation_query.all():
usage = usages[reservation.resource]
if reservation.delta >= 0:
usage.reserved -= reservation.delta
reservation_query.update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')})
def __init__(
self,
user_id,
project_id,
is_admin=None,
read_deleted="no",
roles=None,
remote_address=None,
timestamp=None,
request_id=None,
auth_token=None,
overwrite=True,
quota_class=None,
**kwargs
):
"""
:param read_deleted: 'no' indicates deleted records are hidden, 'yes'
indicates deleted records are visible, 'only' indicates that
*only* deleted records are visible.
:param overwrite: Set to False to ensure that the greenthread local
copy of the index is not overwritten.
:param kwargs: Extra arguments that might be present, but we ignore
because they possibly came in from older rpc messages.
"""
if kwargs:
LOG.warn(_("Arguments dropped when creating context: %s") % str(kwargs))
self.user_id = user_id
self.project_id = project_id
self.roles = roles or []
self.is_admin = is_admin
if self.is_admin is None:
self.is_admin = policy.check_is_admin(self.roles)
elif self.is_admin and "admin" not in self.roles:
self.roles.append("admin")
self.read_deleted = read_deleted
self.remote_address = remote_address
if not timestamp:
timestamp = timeutils.utcnow()
if isinstance(timestamp, basestring):
timestamp = timeutils.parse_strtime(timestamp)
self.timestamp = timestamp
if not request_id:
request_id = generate_request_id()
self.request_id = request_id
self.auth_token = auth_token
self.quota_class = quota_class
if overwrite or not hasattr(local.store, "context"):
self.update_store()
def update_service_capabilities(self, service_name, host, capabilities):
"""Update the per-service capabilities based on this notification."""
if service_name not in ('share'):
LOG.debug('Ignoring %(service_name)s service update '
'from %(host)s', locals())
return
LOG.debug("Received %(service_name)s service update from "
"%(host)s." % {"service_name": service_name, "host": host})
# Copy the capabilities, so we don't modify the original dict
capab_copy = dict(capabilities)
capab_copy["timestamp"] = timeutils.utcnow() # Reported time
self.service_states[host] = capab_copy
def update_service_capabilities(self, service_name, host, capabilities):
"""Update the per-service capabilities based on this notification."""
if service_name not in ('share'):
LOG.debug('Ignoring %(service_name)s service update '
'from %(host)s', locals())
return
LOG.debug("Received %(service_name)s service update from "
"%(host)s." % locals())
# Copy the capabilities, so we don't modify the original dict
capab_copy = dict(capabilities)
capab_copy["timestamp"] = timeutils.utcnow() # Reported time
self.service_states[host] = capab_copy
def create_share(self, context, share_id, request_spec=None,
filter_properties=None, snapshot_id=None):
"""Creates a share."""
context = context.elevated()
if filter_properties is None:
filter_properties = {}
share_ref = self.db.share_get(context, share_id)
if snapshot_id is not None:
snapshot_ref = self.db.share_snapshot_get(context, snapshot_id)
else:
snapshot_ref = None
share_network_id = share_ref.get('share_network_id', None)
if share_network_id:
try:
share_server = self._share_server_get_or_create(
context, share_network_id)
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_("Failed to get share server"
" for share creation."))
self.db.share_update(context, share_id,
{'status': 'error'})
share_ref = self.db.share_update(
context, share_id, {'share_server_id': share_server['id']})
LOG.debug("Using share server %s" % share_server['id'])
else:
share_server = None
try:
if snapshot_ref:
export_location = self.driver.create_share_from_snapshot(
context, share_ref, snapshot_ref)
else:
export_location = self.driver.create_share(
context, share_ref, share_server=share_server)
self.db.share_update(context, share_id,
{'export_location': export_location})
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_("Share %s failed on creation.") % share_id)
self.db.share_update(context, share_id, {'status': 'error'})
else:
LOG.info(_("Share created successfully."))
self.db.share_update(context, share_id,
{'status': 'available',
'launched_at': timeutils.utcnow()})
def _quota_usage_create(context, project_id, user_id, resource, in_use,
reserved, until_refresh, session=None):
quota_usage_ref = models.QuotaUsage()
quota_usage_ref.project_id = project_id
quota_usage_ref.user_id = user_id
quota_usage_ref.resource = resource
quota_usage_ref.in_use = in_use
quota_usage_ref.reserved = reserved
quota_usage_ref.until_refresh = until_refresh
# updated_at is needed for judgement of max_age
quota_usage_ref.updated_at = timeutils.utcnow()
quota_usage_ref.save(session=session)
return quota_usage_ref
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
tests.reset_db()
self.stubs = StubOutForTesting(self)
self.injected = []
self._services = []
CONF.set_override('fatal_exception_format_errors', True)
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
self.log_fixture = self.useFixture(fixtures.FakeLogger())
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(
db_api,
migration,
sql_connection=CONF.database.connection,
sqlite_db=CONF.sqlite_db,
sqlite_clean_db=CONF.sqlite_clean_db,
)
self.useFixture(_DB_CACHE)
self.stubs = StubOutForTesting(self)
self.injected = []
self._services = []
CONF.set_override('fatal_exception_format_errors', True)
rpc.add_extra_exmods('manila.tests')
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
fs = '%(levelname)s [%(name)s] %(message)s'
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
rpc.init(CONF)
fake_notifier.stub_notifier(self.stubs)
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
self.log_fixture = self.useFixture(fixtures.FakeLogger())
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(
db_api,
migration,
sql_connection=CONF.database.connection,
sqlite_db=CONF.sqlite_db,
sqlite_clean_db=CONF.sqlite_clean_db,
)
self.useFixture(_DB_CACHE)
self.stubs = StubOutForTesting(self)
self.injected = []
self._services = []
CONF.set_override('fatal_exception_format_errors', True)
rpc.add_extra_exmods('manila.tests')
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
fs = '%(levelname)s [%(name)s] %(message)s'
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
rpc.init(CONF)
fake_notifier.stub_notifier(self.stubs)
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
tests.reset_db()
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators
self.mox = mox.Mox()
self.stubs = stubout.StubOutForTesting()
self.injected = []
self._services = []
CONF.set_override('fatal_exception_format_errors', True)
def drop_old_duplicate_entries_from_table(migrate_engine, table_name,
use_soft_delete, *uc_column_names):
"""
This method is used to drop all old rows that have the same values for
columns in uc_columns.
"""
meta = MetaData()
meta.bind = migrate_engine
table = Table(table_name, meta, autoload=True)
columns_for_group_by = [table.c[name] for name in uc_column_names]
columns_for_select = [func.max(table.c.id)]
columns_for_select.extend(list(columns_for_group_by))
duplicated_rows_select = select(columns_for_select,
group_by=columns_for_group_by,
having=func.count(table.c.id) > 1)
for row in migrate_engine.execute(duplicated_rows_select):
# NOTE(boris-42): Do not remove row that has the biggest ID.
delete_condition = table.c.id != row[0]
for name in uc_column_names:
delete_condition &= table.c[name] == row[name]
rows_to_delete_select = select([table.c.id]).where(delete_condition)
for row in migrate_engine.execute(rows_to_delete_select).fetchall():
LOG.info(_("Deleted duplicated row with id: %(id)s from table: "
"%(table)s") % dict(id=row[0], table=table_name))
if use_soft_delete:
delete_statement = table.update().\
where(delete_condition).\
values({
'deleted': literal_column('id'),
'updated_at': literal_column('updated_at'),
'deleted_at': timeutils.utcnow()
})
else:
delete_statement = table.delete().where(delete_condition)
migrate_engine.execute(delete_statement)
def create_share(self, context, share_id, request_spec=None,
filter_properties=None, snapshot_id=None):
"""Creates a share."""
context = context.elevated()
if filter_properties is None:
filter_properties = {}
share_ref = self.db.share_get(context, share_id)
if snapshot_id is not None:
snapshot_ref = self.db.share_snapshot_get(context, snapshot_id)
else:
snapshot_ref = None
share_network_id = share_ref.get('share_network_id', None)
if share_network_id:
share_network = self.db.share_network_get(context,
share_network_id)
if share_network['status'] == constants.STATUS_INACTIVE:
share_network = self._activate_share_network(
context,
share_network)
else:
share_network = {}
share_ref['network_info'] = share_network
try:
if snapshot_ref:
export_location = self.driver.create_share_from_snapshot(
context, share_ref, snapshot_ref)
else:
export_location = self.driver.create_share(context, share_ref)
self.db.share_update(context, share_id,
{'export_location': export_location})
except Exception:
with excutils.save_and_reraise_exception():
self.db.share_update(context, share_id, {'status': 'error'})
else:
self.db.share_update(context, share_id,
{'status': 'available',
'launched_at': timeutils.utcnow()})
def delete(self, context, share, force=False):
"""Delete share."""
if context.is_admin and context.project_id != share['project_id']:
project_id = share['project_id']
else:
project_id = context.project_id
share_id = share['id']
if not share['host']:
try:
reservations = QUOTAS.reserve(context,
project_id=project_id,
shares=-1,
gigabytes=-share['size'])
except Exception:
reservations = None
LOG.exception(_("Failed to update quota for deleting share"))
self.db.share_delete(context.elevated(), share_id)
if reservations:
QUOTAS.commit(context, reservations, project_id=project_id)
return
if not (force or share['status'] in ["available", "error"]):
msg = _("Share status must be available or error")
raise exception.InvalidShare(reason=msg)
snapshots = self.db.share_snapshot_get_all_for_share(context, share_id)
if len(snapshots):
msg = _("Share still has %d dependent snapshots") % len(snapshots)
raise exception.InvalidShare(reason=msg)
now = timeutils.utcnow()
share = self.db.share_update(context, share_id, {
'status': 'deleting',
'terminated_at': now
})
self.share_rpcapi.delete_share(context, share)
def index(self, req):
"""
Return a list of all running services. Filter by host & service name.
"""
context = req.environ["manila.context"]
authorize(context)
now = timeutils.utcnow()
services = db.service_get_all(context)
host = ""
if "host" in req.GET:
host = req.GET["host"]
service = ""
if "service" in req.GET:
service = req.GET["service"]
if host:
services = [s for s in services if s["host"] == host]
if service:
services = [s for s in services if s["binary"] == service]
svcs = []
for svc in services:
delta = now - (svc["updated_at"] or svc["created_at"])
alive = abs(utils.total_seconds(delta))
art = (alive and "up") or "down"
active = "enabled"
if svc["disabled"]:
active = "disabled"
svcs.append(
{
"binary": svc["binary"],
"host": svc["host"],
"zone": svc["availability_zone"],
"status": active,
"state": art,
"updated_at": svc["updated_at"],
}
)
return {"services": svcs}
def delete(self, context, share, force=False):
"""Delete share."""
if context.is_admin and context.project_id != share['project_id']:
project_id = share['project_id']
else:
project_id = context.project_id
share_id = share['id']
if not share['host']:
try:
reservations = QUOTAS.reserve(context,
project_id=project_id,
shares=-1,
gigabytes=-share['size'])
except Exception:
reservations = None
LOG.exception(_("Failed to update quota for deleting share"))
self.db.share_delete(context.elevated(), share_id)
if reservations:
QUOTAS.commit(context, reservations, project_id=project_id)
return
if not (force or share['status'] in ["available", "error"]):
msg = _("Share status must be available or error")
raise exception.InvalidShare(reason=msg)
snapshots = self.db.share_snapshot_get_all_for_share(context, share_id)
if len(snapshots):
msg = _("Share still has %d dependent snapshots") % len(snapshots)
raise exception.InvalidShare(reason=msg)
now = timeutils.utcnow()
share = self.db.share_update(context, share_id, {'status': 'deleting',
'terminated_at': now})
self.share_rpcapi.delete_share(context, share)
def create(self,
context,
share_proto,
size,
name,
description,
snapshot=None,
availability_zone=None,
metadata=None,
share_network_id=None,
volume_type=None):
"""Create new share."""
policy.check_policy(context, 'share', 'create')
self._check_metadata_properties(context, metadata)
if snapshot is not None:
if snapshot['status'] != 'available':
msg = _('status must be available')
raise exception.InvalidShareSnapshot(reason=msg)
if not size:
size = snapshot['size']
snapshot_id = snapshot['id']
else:
snapshot_id = None
def as_int(s):
try:
return int(s)
except (ValueError, TypeError):
return s
# tolerate size as stringified int
size = as_int(size)
if not isinstance(size, int) or size <= 0:
msg = (_("Share size '%s' must be an integer and greater than 0") %
size)
raise exception.InvalidInput(reason=msg)
if snapshot and size < snapshot['size']:
msg = (_("Share size '%s' must be equal or greater "
"than snapshot size") % size)
raise exception.InvalidInput(reason=msg)
if snapshot and volume_type:
if volume_type['id'] != snapshot['volume_type_id']:
msg = _("Invalid volume_type provided (requested type "
"must match source snapshot, or be omitted). "
"You should omit the argument.")
raise exception.InvalidInput(reason=msg)
# TODO(rushiagr): Find a suitable place to keep all the allowed
# share types so that it becomes easier to add one
if share_proto.lower() not in ['nfs', 'cifs']:
msg = (_("Invalid share type provided: %s") % share_proto)
raise exception.InvalidInput(reason=msg)
try:
reservations = QUOTAS.reserve(context, shares=1, gigabytes=size)
except exception.OverQuota as e:
overs = e.kwargs['overs']
usages = e.kwargs['usages']
quotas = e.kwargs['quotas']
def _consumed(name):
return (usages[name]['reserved'] + usages[name]['in_use'])
if 'gigabytes' in overs:
msg = _("Quota exceeded for %(s_pid)s, tried to create "
"%(s_size)sG share (%(d_consumed)dG of %(d_quota)dG "
"already consumed)")
LOG.warn(
msg % {
's_pid': context.project_id,
's_size': size,
'd_consumed': _consumed('gigabytes'),
'd_quota': quotas['gigabytes']
})
raise exception.ShareSizeExceedsAvailableQuota()
elif 'shares' in overs:
msg = _("Quota exceeded for %(s_pid)s, tried to create "
"share (%(d_consumed)d shares "
"already consumed)")
LOG.warn(msg % {
's_pid': context.project_id,
'd_consumed': _consumed('shares')
})
raise exception.ShareLimitExceeded(allowed=quotas['shares'])
if availability_zone is None:
availability_zone = CONF.storage_availability_zone
options = {
'size': size,
'user_id': context.user_id,
'project_id': context.project_id,
'snapshot_id': snapshot_id,
'share_network_id': share_network_id,
'availability_zone': availability_zone,
'metadata': metadata,
'status': "creating",
'scheduled_at': timeutils.utcnow(),
'display_name': name,
'display_description': description,
'share_proto': share_proto,
'volume_type_id': volume_type['id'] if volume_type else None
}
try:
share = self.db.share_create(context, options)
QUOTAS.commit(context, reservations)
except Exception:
with excutils.save_and_reraise_exception():
try:
self.db.share_delete(context, share['id'])
finally:
QUOTAS.rollback(context, reservations)
request_spec = {
'share_properties': options,
'share_proto': share_proto,
'share_id': share['id'],
'snapshot_id': share['snapshot_id'],
'volume_type': volume_type
}
filter_properties = {}
self.scheduler_rpcapi.create_share(context,
CONF.share_topic,
share['id'],
snapshot_id,
request_spec=request_spec,
filter_properties=filter_properties)
return share
def create(self, context, share_proto, size, name, description,
snapshot=None, availability_zone=None, metadata=None,
share_network_id=None, volume_type=None):
"""Create new share."""
policy.check_policy(context, 'share', 'create')
self._check_metadata_properties(context, metadata)
if snapshot is not None:
if snapshot['status'] != 'available':
msg = _('status must be available')
raise exception.InvalidShareSnapshot(reason=msg)
if not size:
size = snapshot['size']
snapshot_id = snapshot['id']
else:
snapshot_id = None
def as_int(s):
try:
return int(s)
except (ValueError, TypeError):
return s
# tolerate size as stringified int
size = as_int(size)
if not isinstance(size, int) or size <= 0:
msg = (_("Share size '%s' must be an integer and greater than 0")
% size)
raise exception.InvalidInput(reason=msg)
if snapshot and size < snapshot['size']:
msg = (_("Share size '%s' must be equal or greater "
"than snapshot size") % size)
raise exception.InvalidInput(reason=msg)
if snapshot and volume_type:
if volume_type['id'] != snapshot['volume_type_id']:
msg = _("Invalid volume_type provided (requested type "
"must match source snapshot, or be omitted). "
"You should omit the argument.")
raise exception.InvalidInput(reason=msg)
#TODO(rushiagr): Find a suitable place to keep all the allowed
# share types so that it becomes easier to add one
if share_proto.lower() not in ['nfs', 'cifs']:
msg = (_("Invalid share type provided: %s") % share_proto)
raise exception.InvalidInput(reason=msg)
try:
reservations = QUOTAS.reserve(context, shares=1, gigabytes=size)
except exception.OverQuota as e:
overs = e.kwargs['overs']
usages = e.kwargs['usages']
quotas = e.kwargs['quotas']
def _consumed(name):
return (usages[name]['reserved'] + usages[name]['in_use'])
if 'gigabytes' in overs:
msg = _("Quota exceeded for %(s_pid)s, tried to create "
"%(s_size)sG share (%(d_consumed)dG of %(d_quota)dG "
"already consumed)")
LOG.warn(msg % {'s_pid': context.project_id,
's_size': size,
'd_consumed': _consumed('gigabytes'),
'd_quota': quotas['gigabytes']})
raise exception.ShareSizeExceedsAvailableQuota()
elif 'shares' in overs:
msg = _("Quota exceeded for %(s_pid)s, tried to create "
"share (%(d_consumed)d shares "
"already consumed)")
LOG.warn(msg % {'s_pid': context.project_id,
'd_consumed': _consumed('shares')})
raise exception.ShareLimitExceeded(allowed=quotas['shares'])
if availability_zone is None:
availability_zone = CONF.storage_availability_zone
options = {'size': size,
'user_id': context.user_id,
'project_id': context.project_id,
'snapshot_id': snapshot_id,
'share_network_id': share_network_id,
'availability_zone': availability_zone,
'metadata': metadata,
'status': "creating",
'scheduled_at': timeutils.utcnow(),
'display_name': name,
'display_description': description,
'share_proto': share_proto,
'volume_type_id': volume_type['id'] if volume_type else None
}
try:
share = self.db.share_create(context, options)
QUOTAS.commit(context, reservations)
except Exception:
with excutils.save_and_reraise_exception():
try:
self.db.share_delete(context, share['id'])
finally:
QUOTAS.rollback(context, reservations)
request_spec = {'share_properties': options,
'share_proto': share_proto,
'share_id': share['id'],
'snapshot_id': share['snapshot_id'],
'volume_type': volume_type
}
filter_properties = {}
self.scheduler_rpcapi.create_share(
context,
CONF.share_topic,
share['id'],
snapshot_id,
request_spec=request_spec,
filter_properties=filter_properties)
return share
Fakes For Scheduler tests.
"""
import six
from manila.openstack.common import timeutils
from manila.scheduler import filter_scheduler
from manila.scheduler import host_manager
SHARE_SERVICES = [
dict(id=1,
host='host1',
topic='share',
disabled=False,
availability_zone='zone1',
updated_at=timeutils.utcnow()),
dict(id=2,
host='host2',
topic='share',
disabled=False,
availability_zone='zone1',
updated_at=timeutils.utcnow()),
dict(id=3,
host='host3',
topic='share',
disabled=False,
availability_zone='zone2',
updated_at=timeutils.utcnow()),
dict(id=4,
host='host4',
topic='share',
def auth_token_create(context, token):
fake_token = FakeToken(created_at=timeutils.utcnow(), **token)
FakeAuthDatabase.data[fake_token.token_hash] = fake_token
FakeAuthDatabase.data['id_%i' % fake_token.id] = fake_token
return fake_token
def reserve(self, context, resources, deltas, expire=None,
project_id=None, user_id=None):
"""Check quotas and reserve resources.
For counting quotas--those quotas for which there is a usage
synchronization function--this method checks quotas against
current usage and the desired deltas.
This method will raise a QuotaResourceUnknown exception if a
given resource is unknown or if it does not have a usage
synchronization function.
If any of the proposed values is over the defined quota, an
OverQuota exception will be raised with the sorted list of the
resources which are too high. Otherwise, the method returns a
list of reservation UUIDs which were created.
:param context: The request context, for access checks.
:param resources: A dictionary of the registered resources.
:param deltas: A dictionary of the proposed delta changes.
:param expire: An optional parameter specifying an expiration
time for the reservations. If it is a simple
number, it is interpreted as a number of
seconds and added to the current time; if it is
a datetime.timedelta object, it will also be
added to the current time. A datetime.datetime
object will be interpreted as the absolute
expiration time. If None is specified, the
default expiration time set by
--default-reservation-expire will be used (this
value will be treated as a number of seconds).
:param project_id: Specify the project_id if current context
is admin and admin wants to impact on
common user's tenant.
:param user_id: Specify the user_id if current context
is admin and admin wants to impact on
common user.
"""
# Set up the reservation expiration
if expire is None:
expire = CONF.reservation_expire
if isinstance(expire, (int, long)):
expire = datetime.timedelta(seconds=expire)
if isinstance(expire, datetime.timedelta):
expire = timeutils.utcnow() + expire
if not isinstance(expire, datetime.datetime):
raise exception.InvalidReservationExpiration(expire=expire)
# If project_id is None, then we use the project_id in context
if project_id is None:
project_id = context.project_id
# If user_id is None, then we use the project_id in context
if user_id is None:
user_id = context.user_id
# Get the applicable quotas.
# NOTE(Vek): We're not worried about races at this point.
# Yes, the admin may be in the process of reducing
# quotas, but that's a pretty rare thing.
quotas = self._get_quotas(context, resources, deltas.keys(),
has_sync=True, project_id=project_id)
user_quotas = self._get_quotas(context, resources, deltas.keys(),
has_sync=True, project_id=project_id,
user_id=user_id)
# NOTE(Vek): Most of the work here has to be done in the DB
# API, because we have to do it in a transaction,
# which means access to the session. Since the
# session isn't available outside the DBAPI, we
# have to do the work there.
return db.quota_reserve(context, resources, quotas, user_quotas,
deltas, expire,
CONF.until_refresh, CONF.max_age,
project_id=project_id, user_id=user_id)
def last_completed_audit_period(unit=None):
"""This method gives you the most recently *completed* audit period.
arguments:
units: string, one of 'hour', 'day', 'month', 'year'
Periods normally begin at the beginning (UTC) of the
period unit (So a 'day' period begins at midnight UTC,
a 'month' unit on the 1st, a 'year' on Jan, 1)
unit string may be appended with an optional offset
like so: 'day@18' This will begin the period at 18:00
UTC. 'month@15' starts a monthly period on the 15th,
and year@3 begins a yearly one on March 1st.
returns: 2 tuple of datetimes (begin, end)
The begin timestamp of this audit period is the same as the
end of the previous.
"""
if not unit:
unit = CONF.volume_usage_audit_period
offset = 0
if '@' in unit:
unit, offset = unit.split("@", 1)
offset = int(offset)
rightnow = timeutils.utcnow()
if unit not in ('month', 'day', 'year', 'hour'):
raise ValueError('Time period must be hour, day, month or year')
if unit == 'month':
if offset == 0:
offset = 1
end = datetime.datetime(day=offset,
month=rightnow.month,
year=rightnow.year)
if end >= rightnow:
year = rightnow.year
if 1 >= rightnow.month:
year -= 1
month = 12 + (rightnow.month - 1)
else:
month = rightnow.month - 1
end = datetime.datetime(day=offset, month=month, year=year)
year = end.year
if 1 >= end.month:
year -= 1
month = 12 + (end.month - 1)
else:
month = end.month - 1
begin = datetime.datetime(day=offset, month=month, year=year)
elif unit == 'year':
if offset == 0:
offset = 1
end = datetime.datetime(day=1, month=offset, year=rightnow.year)
if end >= rightnow:
end = datetime.datetime(day=1,
month=offset,
year=rightnow.year - 1)
begin = datetime.datetime(day=1,
month=offset,
year=rightnow.year - 2)
else:
begin = datetime.datetime(day=1,
month=offset,
year=rightnow.year - 1)
elif unit == 'day':
end = datetime.datetime(hour=offset,
day=rightnow.day,
month=rightnow.month,
year=rightnow.year)
if end >= rightnow:
end = end - datetime.timedelta(days=1)
begin = end - datetime.timedelta(days=1)
elif unit == 'hour':
end = rightnow.replace(minute=offset, second=0, microsecond=0)
if end >= rightnow:
end = end - datetime.timedelta(hours=1)
begin = end - datetime.timedelta(hours=1)
return (begin, end)
def service_is_up(service):
"""Check whether a service is up based on last heartbeat."""
last_heartbeat = service['updated_at'] or service['created_at']
# Timestamps in DB are UTC.
elapsed = timeutils.total_seconds(timeutils.utcnow() - last_heartbeat)
return abs(elapsed) <= CONF.service_down_time
def create_share(self,
context,
share_id,
request_spec=None,
filter_properties=None,
snapshot_id=None):
"""Creates a share."""
context = context.elevated()
if filter_properties is None:
filter_properties = {}
share_ref = self.db.share_get(context, share_id)
if snapshot_id is not None:
snapshot_ref = self.db.share_snapshot_get(context, snapshot_id)
parent_share_server_id = snapshot_ref['share']['share_server_id']
else:
snapshot_ref = None
parent_share_server_id = None
share_network_id = share_ref.get('share_network_id', None)
if parent_share_server_id:
try:
share_server = self.db.share_server_get(
context, parent_share_server_id)
LOG.debug("Using share_server "
"%s for share %s" % (share_server['id'], share_id))
share_ref = self.db.share_update(
context, share_id, {'share_server_id': share_server['id']})
except exception.ShareServerNotFound:
with excutils.save_and_reraise_exception():
LOG.error(_("Share server %s does not exist."),
parent_share_server_id)
self.db.share_update(context, share_id,
{'status': 'error'})
elif share_network_id:
try:
share_server, share_ref = self._provide_share_server_for_share(
context, share_network_id, share_id)
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(
_("Failed to get share server"
" for share creation."))
self.db.share_update(context, share_id,
{'status': 'error'})
else:
share_server = None
try:
if snapshot_ref:
export_location = self.driver.create_share_from_snapshot(
context,
share_ref,
snapshot_ref,
share_server=share_server)
else:
export_location = self.driver.create_share(
context, share_ref, share_server=share_server)
self.db.share_update(context, share_id,
{'export_location': export_location})
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_("Share %s failed on creation."), share_id)
self.db.share_update(context, share_id, {'status': 'error'})
else:
LOG.info(_("Share created successfully."))
self.db.share_update(context, share_id, {
'status': 'available',
'launched_at': timeutils.utcnow()
})
def delete(self, session=None):
"""Delete this object."""
self.deleted = self.id
self.deleted_at = timeutils.utcnow()
self.save(session=session)
def _get_time_now(self):
"""Get current UTC. Broken out for testing."""
return timeutils.utcnow()
