def quota_destroy_all_by_project(context, project_id): session = get_session() with session.begin(): model_query(context, models.Quota, session=session, read_deleted="no").\ filter_by(project_id=project_id).\ update({'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at')}, synchronize_session=False) model_query(context, models.ProjectUserQuota, session=session, read_deleted="no").\ filter_by(project_id=project_id).\ update({'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at')}, synchronize_session=False) model_query(context, models.QuotaUsage, session=session, read_deleted="no").\ filter_by(project_id=project_id).\ update({'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at')}, synchronize_session=False) model_query(context, models.Reservation, session=session, read_deleted="no").\ filter_by(project_id=project_id).\ update({'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at')}, synchronize_session=False)
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_LW('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_LE('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def test_share_host_update_db(self): self.mox.StubOutWithMock(timeutils, 'utcnow') self.mox.StubOutWithMock(db, 'share_update') timeutils.utcnow().AndReturn('fake-now') db.share_update(self.context, 31337, {'host': 'fake_host', 'scheduled_at': 'fake-now'}) self.mox.ReplayAll() driver.share_update_db(self.context, 31337, 'fake_host')
def inner(self, *args, **kwargs): LOG.debug("Entering %(cls)s.%(method)s", {'cls': self.__class__.__name__, 'method': func.__name__}) start = timeutils.utcnow() ret = func(self, *args, **kwargs) end = timeutils.utcnow() LOG.debug("Exiting %(cls)s.%(method)s. " "Spent %(duration)s sec. " "Return %(return)s", {'cls': self.__class__.__name__, 'duration': timeutils.delta_seconds(start, end), 'method': func.__name__, 'return': ret}) return ret
def reservation_expire(context): session = get_session() with session.begin(): current_time = timeutils.utcnow() reservation_query = model_query(context, models.Reservation, session=session, read_deleted="no").\ filter(models.Reservation.expire < current_time) for reservation in reservation_query.join(models.QuotaUsage).all(): if reservation.delta >= 0: reservation.usage.reserved -= reservation.delta session.add(reservation.usage) reservation_query.update({'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at')})
def index(self, req): """Return a list of all running services. """ context = req.environ['manila.context'] authorize(context) now = timeutils.utcnow() all_services = db.service_get_all(context) services = [] for service in all_services: service = { 'binary': service['binary'], 'host': service['host'], 'zone': service['availability_zone'], 'status': 'disabled' if service['disabled'] else 'enabled', 'state': 'up' if utils.service_is_up(service) else 'down', 'updated_at': service['updated_at'], } services.append(service) search_opts = [ 'host', 'binary', 'zone', 'state', 'status', ] for search_opt in search_opts: value = '' if search_opt in req.GET: value = req.GET[search_opt] services = [s for s in services if s[search_opt] == value] if len(services) == 0: break return {'services': services}
def create_share(self, context, share_id, request_spec=None, filter_properties=None, snapshot_id=None): """Creates a share.""" context = context.elevated() if filter_properties is None: filter_properties = {} share_ref = self.db.share_get(context, share_id) if snapshot_id is not None: snapshot_ref = self.db.share_snapshot_get(context, snapshot_id) else: snapshot_ref = None try: if snapshot_ref: self.driver.allocate_container_from_snapshot(context, share_ref, snapshot_ref) else: self.driver.allocate_container(context, share_ref) export_location = self.driver.create_share(context, share_ref) self.db.share_update(context, share_id, {'export_location': export_location}) self.driver.create_export(context, share_ref) except Exception: with excutils.save_and_reraise_exception(): self.db.share_update(context, share_id, {'status': 'error'}) else: self.db.share_update(context, share_id, {'status': 'available', 'launched_at': timeutils.utcnow()})
def __init__(self, user_id, project_id, is_admin=None, read_deleted="no", roles=None, remote_address=None, timestamp=None, request_id=None, auth_token=None, overwrite=True, quota_class=None, service_catalog=None, **kwargs): """ :param read_deleted: 'no' indicates deleted records are hidden, 'yes' indicates deleted records are visible, 'only' indicates that *only* deleted records are visible. :param overwrite: Set to False to ensure that the greenthread local copy of the index is not overwritten. :param kwargs: Extra arguments that might be present, but we ignore because they possibly came in from older rpc messages. """ if kwargs: LOG.warn( _('Arguments dropped when creating context: %s') % str(kwargs)) self.user_id = user_id self.project_id = project_id self.roles = roles or [] self.is_admin = is_admin if self.is_admin is None: self.is_admin = policy.check_is_admin(self.roles) elif self.is_admin and 'admin' not in self.roles: self.roles.append('admin') self.read_deleted = read_deleted self.remote_address = remote_address if not timestamp: timestamp = timeutils.utcnow() if isinstance(timestamp, six.string_types): timestamp = timeutils.parse_strtime(timestamp) self.timestamp = timestamp if service_catalog: self.service_catalog = [ s for s in service_catalog if s.get('type') in ('compute', 'volume') ] else: self.service_catalog = [] if not request_id: request_id = generate_request_id() self.request_id = request_id self.auth_token = auth_token self.quota_class = quota_class if overwrite or not hasattr(local.store, 'context'): self.update_store()
def share_update_db(context, share_id, host): '''Set the host and set the scheduled_at field of a share. :returns: A Share with the updated fields set properly. ''' now = timeutils.utcnow() values = {'host': host, 'scheduled_at': now} return db.share_update(context, share_id, values)
def share_delete(context, share_id): session = get_session() share_ref = share_get(context, share_id, session) share_ref.update({'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at'), 'status': 'deleted'}) share_ref.save(session)
def share_update_db(context, share_id, host): '''Set the host and set the scheduled_at field of a share. :returns: A Share with the updated fields set properly. ''' now = timeutils.utcnow() values = {'host': host, 'scheduled_at': now} return db.share_update(context, share_id, values)
def share_access_delete(context, access_id): session = get_session() with session.begin(): session.query(models.ShareAccessMapping).\ filter_by(id=access_id).\ update({'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at'), 'state': models.ShareAccessMapping.STATE_DELETED})
def share_snapshot_destroy(context, snapshot_id): session = get_session() with session.begin(): session.query(models.ShareSnapshot).\ filter_by(id=snapshot_id).\ update({'status': 'deleted', 'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at')})
def inner(self, *args, **kwargs): LOG.debug("Entering %(cls)s.%(method)s", { 'cls': self.__class__.__name__, 'method': func.__name__ }) start = timeutils.utcnow() ret = func(self, *args, **kwargs) end = timeutils.utcnow() LOG.debug( "Exiting %(cls)s.%(method)s. " "Spent %(duration)s sec. " "Return %(return)s", { 'cls': self.__class__.__name__, 'duration': timeutils.delta_seconds(start, end), 'method': func.__name__, 'return': ret }) return ret
def mock_host_manager_db_calls(mock_obj, disabled=None): services = [ dict(id=1, host='host1', topic='share', disabled=False, availability_zone='zone1', updated_at=timeutils.utcnow()), dict(id=2, host='host2', topic='share', disabled=False, availability_zone='zone1', updated_at=timeutils.utcnow()), dict(id=3, host='host3', topic='share', disabled=False, availability_zone='zone2', updated_at=timeutils.utcnow()), dict(id=4, host='host4', topic='share', disabled=False, availability_zone='zone3', updated_at=timeutils.utcnow()), # service on host5 is disabled dict(id=5, host='host5', topic='share', disabled=True, availability_zone='zone4', updated_at=timeutils.utcnow()), ] if disabled is None: mock_obj.return_value = services else: mock_obj.return_value = [service for service in services if service['disabled'] == disabled]
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone, e: self.stop() done.send(e.retvalue)
def notify(context, publisher_id, event_type, priority, payload): """Sends a notification using the specified driver :param publisher_id: the source worker_type.host of the message :param event_type: the literal type of event (ex. Instance Creation) :param priority: patterned after the enumeration of Python logging levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL) :param payload: A python dictionary of attributes Outgoing message format includes the above parameters, and appends the following: message_id a UUID representing the id for this notification timestamp the GMT timestamp the notification was sent at The composite message will be constructed as a dictionary of the above attributes, which will then be sent via the transport mechanism defined by the driver. Message example:: {'message_id': str(uuid.uuid4()), 'publisher_id': 'compute.host1', 'timestamp': timeutils.utcnow(), 'priority': 'WARN', 'event_type': 'compute.create_instance', 'payload': {'instance_id': 12, ... }} """ if priority not in log_levels: raise BadPriorityException(_("%s not in valid priorities") % priority) # Ensure everything is JSON serializable. payload = jsonutils.to_primitive(payload, convert_instances=True) msg = dict( message_id=str(uuid.uuid4()), publisher_id=publisher_id, event_type=event_type, priority=priority, payload=payload, timestamp=str(timeutils.utcnow()), ) for driver in _get_drivers(): try: driver.notify(context, msg) except Exception as e: LOG.exception( _LE("Problem '%(e)s' attempting to " "send to notification system. " "Payload=%(payload)s") % dict(e=e, payload=payload) )
def consume_from_share(self, share): """Incrementally update host state from an share""" share_gb = share['size'] if self.free_capacity_gb == 'infinite': # There's virtually infinite space on back-end pass elif self.free_capacity_gb == 'unknown': # Unable to determine the actual free space on back-end pass else: self.free_capacity_gb -= share_gb self.updated = timeutils.utcnow()
def consume_from_share(self, share): """Incrementally update host state from an share""" share_gb = share['size'] if self.free_capacity_gb == 'infinite': # There's virtually infinite space on back-end pass elif self.free_capacity_gb == 'unknown': # Unable to determine the actual free space on back-end pass else: self.free_capacity_gb -= share_gb self.updated = timeutils.utcnow()
def test_update_service_capabilities_for_shares(self): service_states = self.host_manager.service_states self.assertDictMatch(service_states, {}) self.mox.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().AndReturn(31337) timeutils.utcnow().AndReturn(31338) timeutils.utcnow().AndReturn(31339) host1_share_capabs = dict(free_capacity_gb=4321, timestamp=1) host2_share_capabs = dict(free_capacity_gb=5432, timestamp=1) host3_share_capabs = dict(free_capacity_gb=6543, timestamp=1) self.mox.ReplayAll() service_name = 'share' self.host_manager.update_service_capabilities(service_name, 'host1', host1_share_capabs) self.host_manager.update_service_capabilities(service_name, 'host2', host2_share_capabs) self.host_manager.update_service_capabilities(service_name, 'host3', host3_share_capabs) # Make sure dictionary isn't re-assigned self.assertEqual(self.host_manager.service_states, service_states) # Make sure original dictionary wasn't copied self.assertEqual(host1_share_capabs['timestamp'], 1) host1_share_capabs['timestamp'] = 31337 host2_share_capabs['timestamp'] = 31338 host3_share_capabs['timestamp'] = 31339 expected = {'host1': host1_share_capabs, 'host2': host2_share_capabs, 'host3': host3_share_capabs} self.assertDictMatch(service_states, expected)
def mock_host_manager_db_calls(mock_obj, disabled=None): services = [ dict(id=1, host='host1', topic='share', disabled=False, availability_zone='zone1', updated_at=timeutils.utcnow()), dict(id=2, host='host2', topic='share', disabled=False, availability_zone='zone1', updated_at=timeutils.utcnow()), dict(id=3, host='host3', topic='share', disabled=False, availability_zone='zone2', updated_at=timeutils.utcnow()), dict(id=4, host='host4', topic='share', disabled=False, availability_zone='zone3', updated_at=timeutils.utcnow()), # service on host5 is disabled dict(id=5, host='host5', topic='share', disabled=True, availability_zone='zone4', updated_at=timeutils.utcnow()), ] if disabled is None: mock_obj.return_value = services else: mock_obj.return_value = [ service for service in services if service['disabled'] == disabled ]
def test_service_is_up(self): fts_func = datetime.datetime.fromtimestamp fake_now = 1000 down_time = 5 self.flags(service_down_time=down_time) self.mox.StubOutWithMock(timeutils, 'utcnow') # Up (equal) timeutils.utcnow().AndReturn(fts_func(fake_now)) service = {'updated_at': fts_func(fake_now - down_time), 'created_at': fts_func(fake_now - down_time)} self.mox.ReplayAll() result = utils.service_is_up(service) self.assertTrue(result) self.mox.ResetAll() # Up timeutils.utcnow().AndReturn(fts_func(fake_now)) service = {'updated_at': fts_func(fake_now - down_time + 1), 'created_at': fts_func(fake_now - down_time + 1)} self.mox.ReplayAll() result = utils.service_is_up(service) self.assertTrue(result) self.mox.ResetAll() # Down timeutils.utcnow().AndReturn(fts_func(fake_now)) service = {'updated_at': fts_func(fake_now - down_time - 1), 'created_at': fts_func(fake_now - down_time - 1)} self.mox.ReplayAll() result = utils.service_is_up(service) self.assertFalse(result)
def reservation_rollback(context, reservations, project_id=None, user_id=None): session = get_session() with session.begin(): usages = _get_user_quota_usages(context, session, project_id, user_id) reservation_query = _quota_reservations_query(session, context, reservations) for reservation in reservation_query.all(): usage = usages[reservation.resource] if reservation.delta >= 0: usage.reserved -= reservation.delta reservation_query.update({'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at')})
def __init__( self, user_id, project_id, is_admin=None, read_deleted="no", roles=None, remote_address=None, timestamp=None, request_id=None, auth_token=None, overwrite=True, quota_class=None, **kwargs ): """ :param read_deleted: 'no' indicates deleted records are hidden, 'yes' indicates deleted records are visible, 'only' indicates that *only* deleted records are visible. :param overwrite: Set to False to ensure that the greenthread local copy of the index is not overwritten. :param kwargs: Extra arguments that might be present, but we ignore because they possibly came in from older rpc messages. """ if kwargs: LOG.warn(_("Arguments dropped when creating context: %s") % str(kwargs)) self.user_id = user_id self.project_id = project_id self.roles = roles or [] self.is_admin = is_admin if self.is_admin is None: self.is_admin = policy.check_is_admin(self.roles) elif self.is_admin and "admin" not in self.roles: self.roles.append("admin") self.read_deleted = read_deleted self.remote_address = remote_address if not timestamp: timestamp = timeutils.utcnow() if isinstance(timestamp, basestring): timestamp = timeutils.parse_strtime(timestamp) self.timestamp = timestamp if not request_id: request_id = generate_request_id() self.request_id = request_id self.auth_token = auth_token self.quota_class = quota_class if overwrite or not hasattr(local.store, "context"): self.update_store()
def update_service_capabilities(self, service_name, host, capabilities): """Update the per-service capabilities based on this notification.""" if service_name not in ('share'): LOG.debug('Ignoring %(service_name)s service update ' 'from %(host)s', locals()) return LOG.debug("Received %(service_name)s service update from " "%(host)s." % {"service_name": service_name, "host": host}) # Copy the capabilities, so we don't modify the original dict capab_copy = dict(capabilities) capab_copy["timestamp"] = timeutils.utcnow() # Reported time self.service_states[host] = capab_copy
def update_service_capabilities(self, service_name, host, capabilities): """Update the per-service capabilities based on this notification.""" if service_name not in ('share'): LOG.debug('Ignoring %(service_name)s service update ' 'from %(host)s', locals()) return LOG.debug("Received %(service_name)s service update from " "%(host)s." % locals()) # Copy the capabilities, so we don't modify the original dict capab_copy = dict(capabilities) capab_copy["timestamp"] = timeutils.utcnow() # Reported time self.service_states[host] = capab_copy
def create_share(self, context, share_id, request_spec=None, filter_properties=None, snapshot_id=None): """Creates a share.""" context = context.elevated() if filter_properties is None: filter_properties = {} share_ref = self.db.share_get(context, share_id) if snapshot_id is not None: snapshot_ref = self.db.share_snapshot_get(context, snapshot_id) else: snapshot_ref = None share_network_id = share_ref.get('share_network_id', None) if share_network_id: try: share_server = self._share_server_get_or_create( context, share_network_id) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_("Failed to get share server" " for share creation.")) self.db.share_update(context, share_id, {'status': 'error'}) share_ref = self.db.share_update( context, share_id, {'share_server_id': share_server['id']}) LOG.debug("Using share server %s" % share_server['id']) else: share_server = None try: if snapshot_ref: export_location = self.driver.create_share_from_snapshot( context, share_ref, snapshot_ref) else: export_location = self.driver.create_share( context, share_ref, share_server=share_server) self.db.share_update(context, share_id, {'export_location': export_location}) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_("Share %s failed on creation.") % share_id) self.db.share_update(context, share_id, {'status': 'error'}) else: LOG.info(_("Share created successfully.")) self.db.share_update(context, share_id, {'status': 'available', 'launched_at': timeutils.utcnow()})
def _quota_usage_create(context, project_id, user_id, resource, in_use, reserved, until_refresh, session=None): quota_usage_ref = models.QuotaUsage() quota_usage_ref.project_id = project_id quota_usage_ref.user_id = user_id quota_usage_ref.resource = resource quota_usage_ref.in_use = in_use quota_usage_ref.reserved = reserved quota_usage_ref.until_refresh = until_refresh # updated_at is needed for judgement of max_age quota_usage_ref.updated_at = timeutils.utcnow() quota_usage_ref.save(session=session) return quota_usage_ref
def setUp(self): """Run before each test method to initialize test environment.""" super(TestCase, self).setUp() conf_fixture.set_defaults(CONF) CONF([], default_config_files=[]) # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. self.start = timeutils.utcnow() tests.reset_db() self.stubs = StubOutForTesting(self) self.injected = [] self._services = [] CONF.set_override('fatal_exception_format_errors', True)
def setUp(self): """Run before each test method to initialize test environment.""" super(TestCase, self).setUp() conf_fixture.set_defaults(CONF) CONF([], default_config_files=[]) # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. self.start = timeutils.utcnow() self.log_fixture = self.useFixture(fixtures.FakeLogger()) global _DB_CACHE if not _DB_CACHE: _DB_CACHE = Database( db_api, migration, sql_connection=CONF.database.connection, sqlite_db=CONF.sqlite_db, sqlite_clean_db=CONF.sqlite_clean_db, ) self.useFixture(_DB_CACHE) self.stubs = StubOutForTesting(self) self.injected = [] self._services = [] CONF.set_override('fatal_exception_format_errors', True) rpc.add_extra_exmods('manila.tests') self.addCleanup(rpc.clear_extra_exmods) self.addCleanup(rpc.cleanup) fs = '%(levelname)s [%(name)s] %(message)s' self.messaging_conf = messaging_conffixture.ConfFixture(CONF) self.messaging_conf.transport_driver = 'fake' self.messaging_conf.response_timeout = 15 self.useFixture(self.messaging_conf) rpc.init(CONF) fake_notifier.stub_notifier(self.stubs)
def setUp(self): """Run before each test method to initialize test environment.""" super(TestCase, self).setUp() conf_fixture.set_defaults(CONF) CONF([], default_config_files=[]) # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. self.start = timeutils.utcnow() self.log_fixture = self.useFixture(fixtures.FakeLogger()) global _DB_CACHE if not _DB_CACHE: _DB_CACHE = Database( db_api, migration, sql_connection=CONF.database.connection, sqlite_db=CONF.sqlite_db, sqlite_clean_db=CONF.sqlite_clean_db, ) self.useFixture(_DB_CACHE) self.stubs = StubOutForTesting(self) self.injected = [] self._services = [] CONF.set_override('fatal_exception_format_errors', True) rpc.add_extra_exmods('manila.tests') self.addCleanup(rpc.clear_extra_exmods) self.addCleanup(rpc.cleanup) fs = '%(levelname)s [%(name)s] %(message)s' self.messaging_conf = messaging_conffixture.ConfFixture(CONF) self.messaging_conf.transport_driver = 'fake' self.messaging_conf.response_timeout = 15 self.useFixture(self.messaging_conf) rpc.init(CONF) fake_notifier.stub_notifier(self.stubs)
def setUp(self): """Run before each test method to initialize test environment.""" super(TestCase, self).setUp() conf_fixture.set_defaults(CONF) CONF([], default_config_files=[]) # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. self.start = timeutils.utcnow() tests.reset_db() # emulate some of the mox stuff, we can't use the metaclass # because it screws with our generators self.mox = mox.Mox() self.stubs = stubout.StubOutForTesting() self.injected = [] self._services = [] CONF.set_override('fatal_exception_format_errors', True)
def drop_old_duplicate_entries_from_table(migrate_engine, table_name, use_soft_delete, *uc_column_names): """ This method is used to drop all old rows that have the same values for columns in uc_columns. """ meta = MetaData() meta.bind = migrate_engine table = Table(table_name, meta, autoload=True) columns_for_group_by = [table.c[name] for name in uc_column_names] columns_for_select = [func.max(table.c.id)] columns_for_select.extend(list(columns_for_group_by)) duplicated_rows_select = select(columns_for_select, group_by=columns_for_group_by, having=func.count(table.c.id) > 1) for row in migrate_engine.execute(duplicated_rows_select): # NOTE(boris-42): Do not remove row that has the biggest ID. delete_condition = table.c.id != row[0] for name in uc_column_names: delete_condition &= table.c[name] == row[name] rows_to_delete_select = select([table.c.id]).where(delete_condition) for row in migrate_engine.execute(rows_to_delete_select).fetchall(): LOG.info(_("Deleted duplicated row with id: %(id)s from table: " "%(table)s") % dict(id=row[0], table=table_name)) if use_soft_delete: delete_statement = table.update().\ where(delete_condition).\ values({ 'deleted': literal_column('id'), 'updated_at': literal_column('updated_at'), 'deleted_at': timeutils.utcnow() }) else: delete_statement = table.delete().where(delete_condition) migrate_engine.execute(delete_statement)
def create_share(self, context, share_id, request_spec=None, filter_properties=None, snapshot_id=None): """Creates a share.""" context = context.elevated() if filter_properties is None: filter_properties = {} share_ref = self.db.share_get(context, share_id) if snapshot_id is not None: snapshot_ref = self.db.share_snapshot_get(context, snapshot_id) else: snapshot_ref = None share_network_id = share_ref.get('share_network_id', None) if share_network_id: share_network = self.db.share_network_get(context, share_network_id) if share_network['status'] == constants.STATUS_INACTIVE: share_network = self._activate_share_network( context, share_network) else: share_network = {} share_ref['network_info'] = share_network try: if snapshot_ref: export_location = self.driver.create_share_from_snapshot( context, share_ref, snapshot_ref) else: export_location = self.driver.create_share(context, share_ref) self.db.share_update(context, share_id, {'export_location': export_location}) except Exception: with excutils.save_and_reraise_exception(): self.db.share_update(context, share_id, {'status': 'error'}) else: self.db.share_update(context, share_id, {'status': 'available', 'launched_at': timeutils.utcnow()})
def delete(self, context, share, force=False): """Delete share.""" if context.is_admin and context.project_id != share['project_id']: project_id = share['project_id'] else: project_id = context.project_id share_id = share['id'] if not share['host']: try: reservations = QUOTAS.reserve(context, project_id=project_id, shares=-1, gigabytes=-share['size']) except Exception: reservations = None LOG.exception(_("Failed to update quota for deleting share")) self.db.share_delete(context.elevated(), share_id) if reservations: QUOTAS.commit(context, reservations, project_id=project_id) return if not (force or share['status'] in ["available", "error"]): msg = _("Share status must be available or error") raise exception.InvalidShare(reason=msg) snapshots = self.db.share_snapshot_get_all_for_share(context, share_id) if len(snapshots): msg = _("Share still has %d dependent snapshots") % len(snapshots) raise exception.InvalidShare(reason=msg) now = timeutils.utcnow() share = self.db.share_update(context, share_id, { 'status': 'deleting', 'terminated_at': now }) self.share_rpcapi.delete_share(context, share)
def index(self, req): """ Return a list of all running services. Filter by host & service name. """ context = req.environ["manila.context"] authorize(context) now = timeutils.utcnow() services = db.service_get_all(context) host = "" if "host" in req.GET: host = req.GET["host"] service = "" if "service" in req.GET: service = req.GET["service"] if host: services = [s for s in services if s["host"] == host] if service: services = [s for s in services if s["binary"] == service] svcs = [] for svc in services: delta = now - (svc["updated_at"] or svc["created_at"]) alive = abs(utils.total_seconds(delta)) art = (alive and "up") or "down" active = "enabled" if svc["disabled"]: active = "disabled" svcs.append( { "binary": svc["binary"], "host": svc["host"], "zone": svc["availability_zone"], "status": active, "state": art, "updated_at": svc["updated_at"], } ) return {"services": svcs}
def delete(self, context, share, force=False): """Delete share.""" if context.is_admin and context.project_id != share['project_id']: project_id = share['project_id'] else: project_id = context.project_id share_id = share['id'] if not share['host']: try: reservations = QUOTAS.reserve(context, project_id=project_id, shares=-1, gigabytes=-share['size']) except Exception: reservations = None LOG.exception(_("Failed to update quota for deleting share")) self.db.share_delete(context.elevated(), share_id) if reservations: QUOTAS.commit(context, reservations, project_id=project_id) return if not (force or share['status'] in ["available", "error"]): msg = _("Share status must be available or error") raise exception.InvalidShare(reason=msg) snapshots = self.db.share_snapshot_get_all_for_share(context, share_id) if len(snapshots): msg = _("Share still has %d dependent snapshots") % len(snapshots) raise exception.InvalidShare(reason=msg) now = timeutils.utcnow() share = self.db.share_update(context, share_id, {'status': 'deleting', 'terminated_at': now}) self.share_rpcapi.delete_share(context, share)
def create(self, context, share_proto, size, name, description, snapshot=None, availability_zone=None, metadata=None, share_network_id=None, volume_type=None): """Create new share.""" policy.check_policy(context, 'share', 'create') self._check_metadata_properties(context, metadata) if snapshot is not None: if snapshot['status'] != 'available': msg = _('status must be available') raise exception.InvalidShareSnapshot(reason=msg) if not size: size = snapshot['size'] snapshot_id = snapshot['id'] else: snapshot_id = None def as_int(s): try: return int(s) except (ValueError, TypeError): return s # tolerate size as stringified int size = as_int(size) if not isinstance(size, int) or size <= 0: msg = (_("Share size '%s' must be an integer and greater than 0") % size) raise exception.InvalidInput(reason=msg) if snapshot and size < snapshot['size']: msg = (_("Share size '%s' must be equal or greater " "than snapshot size") % size) raise exception.InvalidInput(reason=msg) if snapshot and volume_type: if volume_type['id'] != snapshot['volume_type_id']: msg = _("Invalid volume_type provided (requested type " "must match source snapshot, or be omitted). " "You should omit the argument.") raise exception.InvalidInput(reason=msg) # TODO(rushiagr): Find a suitable place to keep all the allowed # share types so that it becomes easier to add one if share_proto.lower() not in ['nfs', 'cifs']: msg = (_("Invalid share type provided: %s") % share_proto) raise exception.InvalidInput(reason=msg) try: reservations = QUOTAS.reserve(context, shares=1, gigabytes=size) except exception.OverQuota as e: overs = e.kwargs['overs'] usages = e.kwargs['usages'] quotas = e.kwargs['quotas'] def _consumed(name): return (usages[name]['reserved'] + usages[name]['in_use']) if 'gigabytes' in overs: msg = _("Quota exceeded for %(s_pid)s, tried to create " "%(s_size)sG share (%(d_consumed)dG of %(d_quota)dG " "already consumed)") LOG.warn( msg % { 's_pid': context.project_id, 's_size': size, 'd_consumed': _consumed('gigabytes'), 'd_quota': quotas['gigabytes'] }) raise exception.ShareSizeExceedsAvailableQuota() elif 'shares' in overs: msg = _("Quota exceeded for %(s_pid)s, tried to create " "share (%(d_consumed)d shares " "already consumed)") LOG.warn(msg % { 's_pid': context.project_id, 'd_consumed': _consumed('shares') }) raise exception.ShareLimitExceeded(allowed=quotas['shares']) if availability_zone is None: availability_zone = CONF.storage_availability_zone options = { 'size': size, 'user_id': context.user_id, 'project_id': context.project_id, 'snapshot_id': snapshot_id, 'share_network_id': share_network_id, 'availability_zone': availability_zone, 'metadata': metadata, 'status': "creating", 'scheduled_at': timeutils.utcnow(), 'display_name': name, 'display_description': description, 'share_proto': share_proto, 'volume_type_id': volume_type['id'] if volume_type else None } try: share = self.db.share_create(context, options) QUOTAS.commit(context, reservations) except Exception: with excutils.save_and_reraise_exception(): try: self.db.share_delete(context, share['id']) finally: QUOTAS.rollback(context, reservations) request_spec = { 'share_properties': options, 'share_proto': share_proto, 'share_id': share['id'], 'snapshot_id': share['snapshot_id'], 'volume_type': volume_type } filter_properties = {} self.scheduler_rpcapi.create_share(context, CONF.share_topic, share['id'], snapshot_id, request_spec=request_spec, filter_properties=filter_properties) return share
def create(self, context, share_proto, size, name, description, snapshot=None, availability_zone=None, metadata=None, share_network_id=None, volume_type=None): """Create new share.""" policy.check_policy(context, 'share', 'create') self._check_metadata_properties(context, metadata) if snapshot is not None: if snapshot['status'] != 'available': msg = _('status must be available') raise exception.InvalidShareSnapshot(reason=msg) if not size: size = snapshot['size'] snapshot_id = snapshot['id'] else: snapshot_id = None def as_int(s): try: return int(s) except (ValueError, TypeError): return s # tolerate size as stringified int size = as_int(size) if not isinstance(size, int) or size <= 0: msg = (_("Share size '%s' must be an integer and greater than 0") % size) raise exception.InvalidInput(reason=msg) if snapshot and size < snapshot['size']: msg = (_("Share size '%s' must be equal or greater " "than snapshot size") % size) raise exception.InvalidInput(reason=msg) if snapshot and volume_type: if volume_type['id'] != snapshot['volume_type_id']: msg = _("Invalid volume_type provided (requested type " "must match source snapshot, or be omitted). " "You should omit the argument.") raise exception.InvalidInput(reason=msg) #TODO(rushiagr): Find a suitable place to keep all the allowed # share types so that it becomes easier to add one if share_proto.lower() not in ['nfs', 'cifs']: msg = (_("Invalid share type provided: %s") % share_proto) raise exception.InvalidInput(reason=msg) try: reservations = QUOTAS.reserve(context, shares=1, gigabytes=size) except exception.OverQuota as e: overs = e.kwargs['overs'] usages = e.kwargs['usages'] quotas = e.kwargs['quotas'] def _consumed(name): return (usages[name]['reserved'] + usages[name]['in_use']) if 'gigabytes' in overs: msg = _("Quota exceeded for %(s_pid)s, tried to create " "%(s_size)sG share (%(d_consumed)dG of %(d_quota)dG " "already consumed)") LOG.warn(msg % {'s_pid': context.project_id, 's_size': size, 'd_consumed': _consumed('gigabytes'), 'd_quota': quotas['gigabytes']}) raise exception.ShareSizeExceedsAvailableQuota() elif 'shares' in overs: msg = _("Quota exceeded for %(s_pid)s, tried to create " "share (%(d_consumed)d shares " "already consumed)") LOG.warn(msg % {'s_pid': context.project_id, 'd_consumed': _consumed('shares')}) raise exception.ShareLimitExceeded(allowed=quotas['shares']) if availability_zone is None: availability_zone = CONF.storage_availability_zone options = {'size': size, 'user_id': context.user_id, 'project_id': context.project_id, 'snapshot_id': snapshot_id, 'share_network_id': share_network_id, 'availability_zone': availability_zone, 'metadata': metadata, 'status': "creating", 'scheduled_at': timeutils.utcnow(), 'display_name': name, 'display_description': description, 'share_proto': share_proto, 'volume_type_id': volume_type['id'] if volume_type else None } try: share = self.db.share_create(context, options) QUOTAS.commit(context, reservations) except Exception: with excutils.save_and_reraise_exception(): try: self.db.share_delete(context, share['id']) finally: QUOTAS.rollback(context, reservations) request_spec = {'share_properties': options, 'share_proto': share_proto, 'share_id': share['id'], 'snapshot_id': share['snapshot_id'], 'volume_type': volume_type } filter_properties = {} self.scheduler_rpcapi.create_share( context, CONF.share_topic, share['id'], snapshot_id, request_spec=request_spec, filter_properties=filter_properties) return share
Fakes For Scheduler tests. """ import six from manila.openstack.common import timeutils from manila.scheduler import filter_scheduler from manila.scheduler import host_manager SHARE_SERVICES = [ dict(id=1, host='host1', topic='share', disabled=False, availability_zone='zone1', updated_at=timeutils.utcnow()), dict(id=2, host='host2', topic='share', disabled=False, availability_zone='zone1', updated_at=timeutils.utcnow()), dict(id=3, host='host3', topic='share', disabled=False, availability_zone='zone2', updated_at=timeutils.utcnow()), dict(id=4, host='host4', topic='share',
def auth_token_create(context, token): fake_token = FakeToken(created_at=timeutils.utcnow(), **token) FakeAuthDatabase.data[fake_token.token_hash] = fake_token FakeAuthDatabase.data['id_%i' % fake_token.id] = fake_token return fake_token
def reserve(self, context, resources, deltas, expire=None, project_id=None, user_id=None): """Check quotas and reserve resources. For counting quotas--those quotas for which there is a usage synchronization function--this method checks quotas against current usage and the desired deltas. This method will raise a QuotaResourceUnknown exception if a given resource is unknown or if it does not have a usage synchronization function. If any of the proposed values is over the defined quota, an OverQuota exception will be raised with the sorted list of the resources which are too high. Otherwise, the method returns a list of reservation UUIDs which were created. :param context: The request context, for access checks. :param resources: A dictionary of the registered resources. :param deltas: A dictionary of the proposed delta changes. :param expire: An optional parameter specifying an expiration time for the reservations. If it is a simple number, it is interpreted as a number of seconds and added to the current time; if it is a datetime.timedelta object, it will also be added to the current time. A datetime.datetime object will be interpreted as the absolute expiration time. If None is specified, the default expiration time set by --default-reservation-expire will be used (this value will be treated as a number of seconds). :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. :param user_id: Specify the user_id if current context is admin and admin wants to impact on common user. """ # Set up the reservation expiration if expire is None: expire = CONF.reservation_expire if isinstance(expire, (int, long)): expire = datetime.timedelta(seconds=expire) if isinstance(expire, datetime.timedelta): expire = timeutils.utcnow() + expire if not isinstance(expire, datetime.datetime): raise exception.InvalidReservationExpiration(expire=expire) # If project_id is None, then we use the project_id in context if project_id is None: project_id = context.project_id # If user_id is None, then we use the project_id in context if user_id is None: user_id = context.user_id # Get the applicable quotas. # NOTE(Vek): We're not worried about races at this point. # Yes, the admin may be in the process of reducing # quotas, but that's a pretty rare thing. quotas = self._get_quotas(context, resources, deltas.keys(), has_sync=True, project_id=project_id) user_quotas = self._get_quotas(context, resources, deltas.keys(), has_sync=True, project_id=project_id, user_id=user_id) # NOTE(Vek): Most of the work here has to be done in the DB # API, because we have to do it in a transaction, # which means access to the session. Since the # session isn't available outside the DBAPI, we # have to do the work there. return db.quota_reserve(context, resources, quotas, user_quotas, deltas, expire, CONF.until_refresh, CONF.max_age, project_id=project_id, user_id=user_id)
def last_completed_audit_period(unit=None): """This method gives you the most recently *completed* audit period. arguments: units: string, one of 'hour', 'day', 'month', 'year' Periods normally begin at the beginning (UTC) of the period unit (So a 'day' period begins at midnight UTC, a 'month' unit on the 1st, a 'year' on Jan, 1) unit string may be appended with an optional offset like so: 'day@18' This will begin the period at 18:00 UTC. 'month@15' starts a monthly period on the 15th, and year@3 begins a yearly one on March 1st. returns: 2 tuple of datetimes (begin, end) The begin timestamp of this audit period is the same as the end of the previous. """ if not unit: unit = CONF.volume_usage_audit_period offset = 0 if '@' in unit: unit, offset = unit.split("@", 1) offset = int(offset) rightnow = timeutils.utcnow() if unit not in ('month', 'day', 'year', 'hour'): raise ValueError('Time period must be hour, day, month or year') if unit == 'month': if offset == 0: offset = 1 end = datetime.datetime(day=offset, month=rightnow.month, year=rightnow.year) if end >= rightnow: year = rightnow.year if 1 >= rightnow.month: year -= 1 month = 12 + (rightnow.month - 1) else: month = rightnow.month - 1 end = datetime.datetime(day=offset, month=month, year=year) year = end.year if 1 >= end.month: year -= 1 month = 12 + (end.month - 1) else: month = end.month - 1 begin = datetime.datetime(day=offset, month=month, year=year) elif unit == 'year': if offset == 0: offset = 1 end = datetime.datetime(day=1, month=offset, year=rightnow.year) if end >= rightnow: end = datetime.datetime(day=1, month=offset, year=rightnow.year - 1) begin = datetime.datetime(day=1, month=offset, year=rightnow.year - 2) else: begin = datetime.datetime(day=1, month=offset, year=rightnow.year - 1) elif unit == 'day': end = datetime.datetime(hour=offset, day=rightnow.day, month=rightnow.month, year=rightnow.year) if end >= rightnow: end = end - datetime.timedelta(days=1) begin = end - datetime.timedelta(days=1) elif unit == 'hour': end = rightnow.replace(minute=offset, second=0, microsecond=0) if end >= rightnow: end = end - datetime.timedelta(hours=1) begin = end - datetime.timedelta(hours=1) return (begin, end)
def service_is_up(service): """Check whether a service is up based on last heartbeat.""" last_heartbeat = service['updated_at'] or service['created_at'] # Timestamps in DB are UTC. elapsed = timeutils.total_seconds(timeutils.utcnow() - last_heartbeat) return abs(elapsed) <= CONF.service_down_time
def create_share(self, context, share_id, request_spec=None, filter_properties=None, snapshot_id=None): """Creates a share.""" context = context.elevated() if filter_properties is None: filter_properties = {} share_ref = self.db.share_get(context, share_id) if snapshot_id is not None: snapshot_ref = self.db.share_snapshot_get(context, snapshot_id) parent_share_server_id = snapshot_ref['share']['share_server_id'] else: snapshot_ref = None parent_share_server_id = None share_network_id = share_ref.get('share_network_id', None) if parent_share_server_id: try: share_server = self.db.share_server_get( context, parent_share_server_id) LOG.debug("Using share_server " "%s for share %s" % (share_server['id'], share_id)) share_ref = self.db.share_update( context, share_id, {'share_server_id': share_server['id']}) except exception.ShareServerNotFound: with excutils.save_and_reraise_exception(): LOG.error(_("Share server %s does not exist."), parent_share_server_id) self.db.share_update(context, share_id, {'status': 'error'}) elif share_network_id: try: share_server, share_ref = self._provide_share_server_for_share( context, share_network_id, share_id) except Exception: with excutils.save_and_reraise_exception(): LOG.error( _("Failed to get share server" " for share creation.")) self.db.share_update(context, share_id, {'status': 'error'}) else: share_server = None try: if snapshot_ref: export_location = self.driver.create_share_from_snapshot( context, share_ref, snapshot_ref, share_server=share_server) else: export_location = self.driver.create_share( context, share_ref, share_server=share_server) self.db.share_update(context, share_id, {'export_location': export_location}) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_("Share %s failed on creation."), share_id) self.db.share_update(context, share_id, {'status': 'error'}) else: LOG.info(_("Share created successfully.")) self.db.share_update(context, share_id, { 'status': 'available', 'launched_at': timeutils.utcnow() })
def delete(self, session=None): """Delete this object.""" self.deleted = self.id self.deleted_at = timeutils.utcnow() self.save(session=session)
def _get_time_now(self): """Get current UTC. Broken out for testing.""" return timeutils.utcnow()