def record_metering_data(data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.meter.meter_message_from_counter
"""
session = sqlalchemy_session.get_session()
with session.begin():
if data["source"]:
source = session.query(Source).get(data["source"])
if not source:
source = Source(id=data["source"])
session.add(source)
else:
source = None
# create/update user && project, add/update their sources list
if data["user_id"]:
user = session.merge(User(id=str(data["user_id"])))
if not filter(lambda x: x.id == source.id, user.sources):
user.sources.append(source)
else:
user = None
if data["project_id"]:
project = session.merge(Project(id=str(data["project_id"])))
if not filter(lambda x: x.id == source.id, project.sources):
project.sources.append(source)
else:
project = None
# Record the updated resource metadata
rmetadata = data["resource_metadata"]
resource = session.merge(Resource(id=str(data["resource_id"])))
if not filter(lambda x: x.id == source.id, resource.sources):
resource.sources.append(source)
resource.project = project
resource.user = user
# Current metadata being used and when it was last updated.
resource.resource_metadata = rmetadata
# Record the raw data for the meter.
meter = Meter(
counter_type=data["counter_type"],
counter_unit=data["counter_unit"],
counter_name=data["counter_name"],
resource=resource,
)
session.add(meter)
if not filter(lambda x: x.id == source.id, meter.sources):
meter.sources.append(source)
meter.project = project
meter.user = user
meter.timestamp = data["timestamp"]
meter.resource_metadata = rmetadata
meter.counter_volume = data["counter_volume"]
meter.message_signature = data["message_signature"]
meter.message_id = data["message_id"]
session.flush()
def get_alarms(self, name=None, user=None,
project=None, enabled=True, alarm_id=None,
limit=None, marker_pairs=None,
sort_key=None, sort_dir=None):
"""Yields a lists of alarms that match filters
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param enabled: Optional boolean to list disable alarm.
:param alarm_id: Optional alarm_id to return one alarm.
:param metaquery: Optional dict with metadata to match on.
:param resource: Optional resource filter.
:param limit: Number of documents should be returned.
:param marker_pairs: Attribute-value pairs to identify the last item of
the previous page.
:param sort_key: Attribute by which results be sorted.
:param sort_dir: Direction with which results be sorted(asc, desc).
"""
if limit or marker_pairs or sort_key or sort_dir:
raise NotImplementedError(_('Pagination not implemented'))
session = sqlalchemy_session.get_session()
query = session.query(Alarm)
if name is not None:
query = query.filter(Alarm.name == name)
if enabled is not None:
query = query.filter(Alarm.enabled == enabled)
if user is not None:
query = query.filter(Alarm.user_id == user)
if project is not None:
query = query.filter(Alarm.project_id == project)
if alarm_id is not None:
query = query.filter(Alarm.id == alarm_id)
return (self._row_to_alarm_model(x) for x in query.all())
def get_trait_types(event_type):
"""Return a dictionary containing the name and data type of
the trait type. Only trait types for the provided event_type are
returned.
:param event_type: the type of the Event
"""
session = sqlalchemy_session.get_session()
LOG.debug(_("Get traits for %s") % event_type)
with session.begin():
query = (session.query(models.TraitType.desc,
models.TraitType.data_type)
.join(models.Trait,
models.Trait.trait_type_id ==
models.TraitType.id)
.join(models.Event,
models.Event.id ==
models.Trait.event_id)
.join(models.EventType,
and_(models.EventType.id ==
models.Event.id,
models.EventType.desc ==
event_type))
.group_by(models.TraitType.desc,
models.TraitType.data_type)
.distinct())
for desc, type in query.all():
yield {'name': desc, 'data_type': type}
def get_meters(user=None, project=None, resource=None, source=None, metaquery={}, pagination=None):
"""Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError(_("Pagination not implemented"))
if metaquery:
raise NotImplementedError(_("metaquery not implemented"))
session = sqlalchemy_session.get_session()
# Meter table will store large records and join with resource
# will be very slow.
# subquery_meter is used to reduce meter records
# by selecting a record for each (resource_id, counter_name).
# max() is used to choice a meter record, so the latest record
# is selected for each (resource_id, counter_name).
#
subquery_meter = (
session.query(func.max(Meter.id).label("id")).group_by(Meter.resource_id, Meter.counter_name).subquery()
)
# The SQL of query_meter is essentially:
#
# SELECT meter.* FROM meter INNER JOIN
# (SELECT max(meter.id) AS id FROM meter
# GROUP BY meter.resource_id, meter.counter_name) AS anon_2
# ON meter.id = anon_2.id
#
query_meter = session.query(Meter).join(subquery_meter, Meter.id == subquery_meter.c.id)
alias_meter = aliased(Meter, query_meter.subquery())
query = session.query(Resource, alias_meter).join(alias_meter, Resource.id == alias_meter.resource_id)
if user is not None:
query = query.filter(Resource.user_id == user)
if source is not None:
query = query.filter(Resource.sources.any(id=source))
if resource:
query = query.filter(Resource.id == resource)
if project is not None:
query = query.filter(Resource.project_id == project)
for resource, meter in query.all():
yield api_models.Meter(
name=meter.counter_name,
type=meter.counter_type,
unit=meter.counter_unit,
resource_id=resource.id,
project_id=resource.project_id,
source=resource.sources[0].id,
user_id=resource.user_id,
)
def clear_expired_metering_data(ttl):
"""Clear expired data from the backend storage system according to the
time-to-live.
:param ttl: Number of seconds to keep records for.
"""
session = sqlalchemy_session.get_session()
query = session.query(Meter.id)
end = timeutils.utcnow() - datetime.timedelta(seconds=ttl)
query = query.filter(Meter.timestamp < end)
query.delete()
query = session.query(User.id).filter(~User.id.in_(
session.query(Meter.user_id).group_by(Meter.user_id)
))
query.delete(synchronize_session='fetch')
query = session.query(Project.id).filter(~Project.id.in_(
session.query(Meter.project_id).group_by(Meter.project_id)
))
query.delete(synchronize_session='fetch')
query = session.query(Resource.id).filter(~Resource.id.in_(
session.query(Meter.resource_id).group_by(Meter.resource_id)
))
query.delete(synchronize_session='fetch')
def get_traits(event_type, trait_type=None):
"""Return all trait instances associated with an event_type. If
trait_type is specified, only return instances of that trait type.
:param event_type: the type of the Event to filter by
:param trait_type: the name of the Trait to filter by
"""
session = sqlalchemy_session.get_session()
with session.begin():
trait_type_filters = [models.TraitType.id ==
models.Trait.trait_type_id]
if trait_type:
trait_type_filters.append(models.TraitType.desc == trait_type)
query = (session.query(models.Trait)
.join(models.TraitType, and_(*trait_type_filters))
.join(models.Event,
models.Event.id == models.Trait.event_id)
.join(models.EventType,
and_(models.EventType.id ==
models.Event.event_type_id,
models.EventType.desc == event_type)))
for trait in query.all():
type = trait.trait_type
yield api_models.Trait(name=type.desc,
dtype=type.data_type,
value=trait.get_value())
def record_metering_data(cls, data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.meter.meter_message_from_counter
"""
session = sqlalchemy_session.get_session()
with session.begin():
# Record the updated resource metadata
rmetadata = data['resource_metadata']
source = cls._create_or_update(session, Source, data['source'])
user = cls._create_or_update(session, User, data['user_id'],
source)
project = cls._create_or_update(session, Project,
data['project_id'], source)
resource = cls._create_or_update(session, Resource,
data['resource_id'], source,
user=user, project=project,
resource_metadata=rmetadata)
# Record the raw data for the meter.
meter = Meter(counter_type=data['counter_type'],
counter_unit=data['counter_unit'],
counter_name=data['counter_name'], resource=resource)
session.add(meter)
if not filter(lambda x: x.id == source.id, meter.sources):
meter.sources.append(source)
meter.project = project
meter.user = user
meter.timestamp = data['timestamp']
meter.resource_metadata = rmetadata
meter.counter_volume = data['counter_volume']
meter.message_signature = data['message_signature']
meter.message_id = data['message_id']
def get_alarm_changes(
self,
alarm_id,
on_behalf_of,
user=None,
project=None,
type=None,
start_timestamp=None,
start_timestamp_op=None,
end_timestamp=None,
end_timestamp_op=None,
):
"""Yields list of AlarmChanges describing alarm history
Changes are always sorted in reverse order of occurence, given
the importance of currency.
Segregation for non-administrative users is done on the basis
of the on_behalf_of parameter. This allows such users to have
visibility on both the changes initiated by themselves directly
(generally creation, rule changes, or deletion) and also on those
changes initiated on their behalf by the alarming service (state
transitions after alarm thresholds are crossed).
:param alarm_id: ID of alarm to return changes for
:param on_behalf_of: ID of tenant to scope changes query (None for
administrative user, indicating all projects)
:param user: Optional ID of user to return changes for
:param project: Optional ID of project to return changes for
:project type: Optional change type
:param start_timestamp: Optional modified timestamp start range
:param start_timestamp_op: Optional timestamp start range operation
:param end_timestamp: Optional modified timestamp end range
:param end_timestamp_op: Optional timestamp end range operation
"""
session = sqlalchemy_session.get_session()
query = session.query(AlarmChange)
query = query.filter(AlarmChange.alarm_id == alarm_id)
if on_behalf_of is not None:
query = query.filter(AlarmChange.on_behalf_of == on_behalf_of)
if user is not None:
query = query.filter(AlarmChange.user_id == user)
if project is not None:
query = query.filter(AlarmChange.project_id == project)
if type is not None:
query = query.filter(AlarmChange.type == type)
if start_timestamp:
if start_timestamp_op == "gt":
query = query.filter(AlarmChange.timestamp > start_timestamp)
else:
query = query.filter(AlarmChange.timestamp >= start_timestamp)
if end_timestamp:
if end_timestamp_op == "le":
query = query.filter(AlarmChange.timestamp <= end_timestamp)
else:
query = query.filter(AlarmChange.timestamp < end_timestamp)
query = query.order_by(desc(AlarmChange.timestamp))
return (self._row_to_alarm_change_model(x) for x in query.all())
def get_alarms(self,
name=None,
user=None,
project=None,
enabled=None,
alarm_id=None,
pagination=None):
"""Yields a lists of alarms that match filters
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param enabled: Optional boolean to list disable alarm.
:param alarm_id: Optional alarm_id to return one alarm.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError(_('Pagination not implemented'))
session = sqlalchemy_session.get_session()
query = session.query(Alarm)
if name is not None:
query = query.filter(Alarm.name == name)
if enabled is not None:
query = query.filter(Alarm.enabled == enabled)
if user is not None:
query = query.filter(Alarm.user_id == user)
if project is not None:
query = query.filter(Alarm.project_id == project)
if alarm_id is not None:
query = query.filter(Alarm.id == alarm_id)
return (self._row_to_alarm_model(x) for x in query.all())
def get_alarms(self, name=None, user=None,
project=None, enabled=None, alarm_id=None, pagination=None):
"""Yields a lists of alarms that match filters
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param enabled: Optional boolean to list disable alarm.
:param alarm_id: Optional alarm_id to return one alarm.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError(_('Pagination not implemented'))
session = sqlalchemy_session.get_session()
query = session.query(Alarm)
if name is not None:
query = query.filter(Alarm.name == name)
if enabled is not None:
query = query.filter(Alarm.enabled == enabled)
if user is not None:
query = query.filter(Alarm.user_id == user)
if project is not None:
query = query.filter(Alarm.project_id == project)
if alarm_id is not None:
query = query.filter(Alarm.id == alarm_id)
return (self._row_to_alarm_model(x) for x in query.all())
def get_traits(event_type, trait_type=None):
"""Return all trait instances associated with an event_type. If
trait_type is specified, only return instances of that trait type.
:param event_type: the type of the Event to filter by
:param trait_type: the name of the Trait to filter by
"""
session = sqlalchemy_session.get_session()
with session.begin():
trait_type_filters = [
models.TraitType.id == models.Trait.trait_type_id
]
if trait_type:
trait_type_filters.append(models.TraitType.desc == trait_type)
query = (session.query(models.Trait).join(
models.TraitType, and_(*trait_type_filters)).join(
models.Event,
models.Event.id == models.Trait.event_id).join(
models.EventType,
and_(models.EventType.id == models.Event.event_type_id,
models.EventType.desc == event_type)))
for trait in query.all():
type = trait.trait_type
yield api_models.Trait(name=type.desc,
dtype=type.data_type,
value=trait.get_value())
def get_trait_types(event_type):
"""Return a dictionary containing the name and data type of
the trait type. Only trait types for the provided event_type are
returned.
:param event_type: the type of the Event
"""
session = sqlalchemy_session.get_session()
LOG.debug(_("Get traits for %s") % event_type)
with session.begin():
query = (session.query(
models.TraitType.desc, models.TraitType.data_type).join(
models.Trait,
models.Trait.trait_type_id == models.TraitType.id).join(
models.Event,
models.Event.id == models.Trait.event_id).join(
models.EventType,
and_(
models.EventType.id == models.Event.id,
models.EventType.desc == event_type)).group_by(
models.TraitType.desc,
models.TraitType.data_type).distinct())
for desc, type in query.all():
yield {'name': desc, 'data_type': type}
def clear_expired_metering_data(ttl):
"""Clear expired data from the backend storage system according to the
time-to-live.
:param ttl: Number of seconds to keep records for.
"""
session = sqlalchemy_session.get_session()
query = session.query(Meter.id)
end = timeutils.utcnow() - datetime.timedelta(seconds=ttl)
query = query.filter(Meter.timestamp < end)
query.delete()
query = session.query(User.id).filter(~User.id.in_(
session.query(Meter.user_id).group_by(Meter.user_id)
))
query.delete(synchronize_session='fetch')
query = session.query(Project.id).filter(~Project.id.in_(
session.query(Meter.project_id).group_by(Meter.project_id)
))
query.delete(synchronize_session='fetch')
query = session.query(Resource.id).filter(~Resource.id.in_(
session.query(Meter.resource_id).group_by(Meter.resource_id)
))
query.delete(synchronize_session='fetch')
def record_events(self, event_models):
"""Write the events to SQL database via sqlalchemy.
:param event_models: a list of model.Event objects.
Returns a list of events that could not be saved in a
(reason, event) tuple. Reasons are enumerated in
storage.model.Event
"""
session = sqlalchemy_session.get_session()
events = []
problem_events = []
for event_model in event_models:
event = None
try:
with session.begin():
event = self._record_event(session, event_model)
session.flush()
except dbexc.DBDuplicateEntry:
problem_events.append(
(api_models.Event.DUPLICATE, event_model))
except Exception as e:
LOG.exception('Failed to record event: %s', e)
problem_events.append(
(api_models.Event.UNKNOWN_PROBLEM, event_model))
events.append(event)
return problem_events
def record_events(self, event_models):
"""Write the events to SQL database via sqlalchemy.
:param event_models: a list of model.Event objects.
Returns a list of events that could not be saved in a
(reason, event) tuple. Reasons are enumerated in
storage.model.Event
"""
session = sqlalchemy_session.get_session()
events = []
problem_events = []
for event_model in event_models:
event = None
try:
with session.begin():
event = self._record_event(session, event_model)
session.flush()
except dbexc.DBDuplicateEntry:
problem_events.append((api_models.Event.DUPLICATE,
event_model))
except Exception as e:
LOG.exception('Failed to record event: %s', e)
problem_events.append((api_models.Event.UNKNOWN_PROBLEM,
event_model))
events.append(event)
return problem_events
def get_resources(user=None,
project=None,
source=None,
start_timestamp=None,
start_timestamp_op=None,
end_timestamp=None,
end_timestamp_op=None,
metaquery={},
resource=None):
"""Return an iterable of api_models.Resource instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param start_timestamp_op: Optonal start time operator, like gt, ge.
:param end_timestamp: Optional modified timestamp end range.
:param end_timestamp_op: Optional end time operator, like lt, le.
:param metaquery: Optional dict with metadata to match on.
:param resource: Optional resource filter.
"""
session = sqlalchemy_session.get_session()
query = session.query(Meter, ).group_by(Meter.resource_id)
if user is not None:
query = query.filter(Meter.user_id == user)
if source is not None:
query = query.filter(Meter.sources.any(id=source))
if start_timestamp:
if start_timestamp_op == 'gt':
query = query.filter(Meter.timestamp > start_timestamp)
else:
query = query.filter(Meter.timestamp >= start_timestamp)
if end_timestamp:
if end_timestamp_op == 'le':
query = query.filter(Meter.timestamp <= end_timestamp)
else:
query = query.filter(Meter.timestamp < end_timestamp)
if project is not None:
query = query.filter(Meter.project_id == project)
if resource is not None:
query = query.filter(Meter.resource_id == resource)
if metaquery:
raise NotImplementedError('metaquery not implemented')
for meter in query.all():
yield api_models.Resource(
resource_id=meter.resource_id,
project_id=meter.project_id,
source=meter.sources[0].id,
user_id=meter.user_id,
metadata=meter.resource_metadata,
meter=[
api_models.ResourceMeter(
counter_name=m.counter_name,
counter_type=m.counter_type,
counter_unit=m.counter_unit,
) for m in meter.resource.meters
],
)
def get_alarm_changes(self,
alarm_id,
on_behalf_of,
user=None,
project=None,
type=None,
start_timestamp=None,
start_timestamp_op=None,
end_timestamp=None,
end_timestamp_op=None):
"""Yields list of AlarmChanges describing alarm history
Changes are always sorted in reverse order of occurence, given
the importance of currency.
Segregation for non-administrative users is done on the basis
of the on_behalf_of parameter. This allows such users to have
visibility on both the changes initiated by themselves directly
(generally creation, rule changes, or deletion) and also on those
changes initiated on their behalf by the alarming service (state
transitions after alarm thresholds are crossed).
:param alarm_id: ID of alarm to return changes for
:param on_behalf_of: ID of tenant to scope changes query (None for
administrative user, indicating all projects)
:param user: Optional ID of user to return changes for
:param project: Optional ID of project to return changes for
:project type: Optional change type
:param start_timestamp: Optional modified timestamp start range
:param start_timestamp_op: Optional timestamp start range operation
:param end_timestamp: Optional modified timestamp end range
:param end_timestamp_op: Optional timestamp end range operation
"""
session = sqlalchemy_session.get_session()
query = session.query(AlarmChange)
query = query.filter(AlarmChange.alarm_id == alarm_id)
if on_behalf_of is not None:
query = query.filter(AlarmChange.on_behalf_of == on_behalf_of)
if user is not None:
query = query.filter(AlarmChange.user_id == user)
if project is not None:
query = query.filter(AlarmChange.project_id == project)
if type is not None:
query = query.filter(AlarmChange.type == type)
if start_timestamp:
if start_timestamp_op == 'gt':
query = query.filter(AlarmChange.timestamp > start_timestamp)
else:
query = query.filter(AlarmChange.timestamp >= start_timestamp)
if end_timestamp:
if end_timestamp_op == 'le':
query = query.filter(AlarmChange.timestamp <= end_timestamp)
else:
query = query.filter(AlarmChange.timestamp < end_timestamp)
query = query.order_by(desc(AlarmChange.timestamp))
return (self._row_to_alarm_change_model(x) for x in query.all())
def record_metering_data(data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.meter.meter_message_from_counter
"""
session = sqlalchemy_session.get_session()
with session.begin():
if data['source']:
source = session.query(Source).get(data['source'])
if not source:
source = Source(id=data['source'])
session.add(source)
else:
source = None
# create/update user && project, add/update their sources list
if data['user_id']:
user = session.merge(User(id=str(data['user_id'])))
if not filter(lambda x: x.id == source.id, user.sources):
user.sources.append(source)
else:
user = None
if data['project_id']:
project = session.merge(Project(id=str(data['project_id'])))
if not filter(lambda x: x.id == source.id, project.sources):
project.sources.append(source)
else:
project = None
# Record the updated resource metadata
rmetadata = data['resource_metadata']
resource = session.merge(Resource(id=str(data['resource_id'])))
if not filter(lambda x: x.id == source.id, resource.sources):
resource.sources.append(source)
resource.project = project
resource.user = user
# Current metadata being used and when it was last updated.
resource.resource_metadata = rmetadata
# Record the raw data for the meter.
meter = Meter(counter_type=data['counter_type'],
counter_unit=data['counter_unit'],
counter_name=data['counter_name'],
resource=resource)
session.add(meter)
if not filter(lambda x: x.id == source.id, meter.sources):
meter.sources.append(source)
meter.project = project
meter.user = user
meter.timestamp = data['timestamp']
meter.resource_metadata = rmetadata
meter.counter_volume = data['counter_volume']
meter.message_signature = data['message_signature']
meter.message_id = data['message_id']
session.flush()
def get_meters(user=None,
project=None,
resource=None,
source=None,
metaquery={}):
"""Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
session = sqlalchemy_session.get_session()
# Meter table will store large records and join with resource
# will be very slow.
# subquery_meter is used to reduce meter records
# by selecting a record for each (resource_id, counter_name).
# max() is used to choice a meter record, so the latest record
# is selected for each (resource_id, counter_name).
#
subquery_meter = session.query(func.max(Meter.id).label('id')).\
group_by(Meter.resource_id, Meter.counter_name).subquery()
# The SQL of query_meter is essentially:
#
# SELECT meter.* FROM meter INNER JOIN
# (SELECT max(meter.id) AS id FROM meter
# GROUP BY meter.resource_id, meter.counter_name) AS anon_2
# ON meter.id = anon_2.id
#
query_meter = session.query(Meter).\
join(subquery_meter, Meter.id == subquery_meter.c.id)
alias_meter = aliased(Meter, query_meter.subquery())
query = session.query(Resource, alias_meter).join(
alias_meter, Resource.id == alias_meter.resource_id)
if user is not None:
query = query.filter(Resource.user_id == user)
if source is not None:
query = query.filter(Resource.sources.any(id=source))
if resource:
query = query.filter(Resource.id == resource)
if project is not None:
query = query.filter(Resource.project_id == project)
if metaquery:
raise NotImplementedError('metaquery not implemented')
for resource, meter in query.all():
yield api_models.Meter(name=meter.counter_name,
type=meter.counter_type,
unit=meter.counter_unit,
resource_id=resource.id,
project_id=resource.project_id,
source=resource.sources[0].id,
user_id=resource.user_id)
def delete_alarm(alarm_id):
"""Delete a alarm
:param alarm_id: ID of the alarm to delete
"""
session = sqlalchemy_session.get_session()
with session.begin():
session.query(Alarm).filter(Alarm.id == alarm_id).delete()
session.flush()
def _make_volume_query(sample_filter, counter_volume_func):
"""Returns complex Meter counter_volume query for max and sum."""
session = sqlalchemy_session.get_session()
subq = session.query(Meter.id)
subq = make_query_from_filter(subq, sample_filter, require_meter=False)
subq = subq.subquery()
mainq = session.query(Resource.id, counter_volume_func)
mainq = mainq.join(Meter).group_by(Resource.id)
return mainq.filter(Meter.id.in_(subq))
def delete_alarm(alarm_id):
"""Delete a alarm
:param alarm_id: ID of the alarm to delete
"""
session = sqlalchemy_session.get_session()
with session.begin():
session.query(Alarm).filter(Alarm.id == alarm_id).delete()
session.flush()
def _make_volume_query(sample_filter, counter_volume_func):
"""Returns complex Meter counter_volume query for max and sum."""
session = sqlalchemy_session.get_session()
subq = session.query(Meter.id)
subq = make_query_from_filter(subq, sample_filter, require_meter=False)
subq = subq.subquery()
mainq = session.query(Resource.id, counter_volume_func)
mainq = mainq.join(Meter).group_by(Resource.id)
return mainq.filter(Meter.id.in_(subq))
def get_users(source=None):
"""Return an iterable of user id strings.
:param source: Optional source filter.
"""
session = sqlalchemy_session.get_session()
query = session.query(User.id)
if source is not None:
query = query.filter(User.sources.any(id=source))
return (x[0] for x in query.all())
def get_projects(source=None):
"""Return an iterable of project id strings.
:param source: Optional source filter.
"""
session = sqlalchemy_session.get_session()
query = session.query(Project.id)
if source:
query = query.filter(Project.sources.any(id=source))
return (x[0] for x in query.all())
def get_projects(source=None):
"""Return an iterable of project id strings.
:param source: Optional source filter.
"""
session = sqlalchemy_session.get_session()
query = session.query(Project.id)
if source:
query = query.filter(Project.sources.any(id=source))
return (x[0] for x in query.all())
def get_users(source=None):
"""Return an iterable of user id strings.
:param source: Optional source filter.
"""
session = sqlalchemy_session.get_session()
query = session.query(User.id)
if source is not None:
query = query.filter(User.sources.any(id=source))
return (x[0] for x in query.all())
def get_events(self, event_filter):
"""Return an iterable of model.Event objects.
:param event_filter: EventFilter instance
"""
start = utils.dt_to_decimal(event_filter.start)
end = utils.dt_to_decimal(event_filter.end)
session = sqlalchemy_session.get_session()
with session.begin():
sub_query = session.query(Event.id)\
.join(Trait, Trait.event_id == Event.id)\
.filter(Event.generated >= start, Event.generated <= end)
if event_filter.event_name:
event_name = self._get_unique(session, event_filter.event_name)
sub_query = sub_query.filter(Event.unique_name == event_name)
if event_filter.traits:
for key, value in event_filter.traits.iteritems():
if key == 'key':
key = self._get_unique(session, value)
sub_query = sub_query.filter(Trait.name == key)
elif key == 't_string':
sub_query = sub_query.filter(Trait.t_string == value)
elif key == 't_int':
sub_query = sub_query.filter(Trait.t_int == value)
elif key == 't_datetime':
dt = utils.dt_to_decimal(value)
sub_query = sub_query.filter(Trait.t_datetime == dt)
elif key == 't_float':
sub_query = sub_query.filter(Trait.t_datetime == value)
sub_query = sub_query.subquery()
all_data = session.query(Trait)\
.join(sub_query, Trait.event_id == sub_query.c.id)
# Now convert the sqlalchemy objects back into Models ...
event_models_dict = {}
for trait in all_data.all():
event = event_models_dict.get(trait.event_id)
if not event:
generated = utils.decimal_to_dt(trait.event.generated)
event = api_models.Event(trait.event.message_id,
trait.event.unique_name.key,
generated, [])
event_models_dict[trait.event_id] = event
value = trait.get_value()
trait_model = api_models.Trait(trait.name.key, trait.t_type,
value)
event.append_trait(trait_model)
event_models = event_models_dict.values()
return sorted(event_models, key=operator.attrgetter('generated'))
def get_resources(user=None, project=None, source=None,
start_timestamp=None, start_timestamp_op=None,
end_timestamp=None, end_timestamp_op=None,
metaquery={}, resource=None):
"""Return an iterable of api_models.Resource instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param start_timestamp_op: Optonal start time operator, like gt, ge.
:param end_timestamp: Optional modified timestamp end range.
:param end_timestamp_op: Optional end time operator, like lt, le.
:param metaquery: Optional dict with metadata to match on.
:param resource: Optional resource filter.
"""
session = sqlalchemy_session.get_session()
query = session.query(Meter,).group_by(Meter.resource_id)
if user is not None:
query = query.filter(Meter.user_id == user)
if source is not None:
query = query.filter(Meter.sources.any(id=source))
if start_timestamp:
if start_timestamp_op == 'gt':
query = query.filter(Meter.timestamp > start_timestamp)
else:
query = query.filter(Meter.timestamp >= start_timestamp)
if end_timestamp:
if end_timestamp_op == 'le':
query = query.filter(Meter.timestamp <= end_timestamp)
else:
query = query.filter(Meter.timestamp < end_timestamp)
if project is not None:
query = query.filter(Meter.project_id == project)
if resource is not None:
query = query.filter(Meter.resource_id == resource)
if metaquery:
raise NotImplementedError('metaquery not implemented')
for meter in query.all():
yield api_models.Resource(
resource_id=meter.resource_id,
project_id=meter.project_id,
source=meter.sources[0].id,
user_id=meter.user_id,
metadata=meter.resource_metadata,
meter=[
api_models.ResourceMeter(
counter_name=m.counter_name,
counter_type=m.counter_type,
counter_unit=m.counter_unit,
)
for m in meter.resource.meters
],
)
def get_event_types():
"""Return all event types as an iterable of strings.
"""
session = sqlalchemy_session.get_session()
with session.begin():
query = session.query(models.EventType.desc)\
.order_by(models.EventType.desc)
for name in query.all():
# The query returns a tuple with one element.
yield name[0]
def update_alarm(self, alarm):
"""Update an alarm.
:param alarm: the new Alarm to update
"""
session = sqlalchemy_session.get_session()
with session.begin():
alarm_row = session.merge(models.Alarm(id=alarm.alarm_id))
alarm_row.update(alarm.as_dict())
return self._row_to_alarm_model(alarm_row)
def get_event_types():
"""Return all event types as an iterable of strings.
"""
session = sqlalchemy_session.get_session()
with session.begin():
query = session.query(models.EventType.desc)\
.order_by(models.EventType.desc)
for name in query.all():
# The query returns a tuple with one element.
yield name[0]
def get_events(self, event_filter):
"""Return an iterable of model.Event objects.
:param event_filter: EventFilter instance
"""
start = utils.dt_to_decimal(event_filter.start)
end = utils.dt_to_decimal(event_filter.end)
session = sqlalchemy_session.get_session()
with session.begin():
sub_query = session.query(Event.id)\
.join(Trait, Trait.event_id == Event.id)\
.filter(Event.generated >= start, Event.generated <= end)
if event_filter.event_name:
event_name = self._get_unique(session, event_filter.event_name)
sub_query = sub_query.filter(Event.unique_name == event_name)
if event_filter.traits:
for key, value in event_filter.traits.iteritems():
if key == 'key':
key = self._get_unique(session, value)
sub_query = sub_query.filter(Trait.name == key)
elif key == 't_string':
sub_query = sub_query.filter(Trait.t_string == value)
elif key == 't_int':
sub_query = sub_query.filter(Trait.t_int == value)
elif key == 't_datetime':
dt = utils.dt_to_decimal(value)
sub_query = sub_query.filter(Trait.t_datetime == dt)
elif key == 't_float':
sub_query = sub_query.filter(Trait.t_datetime == value)
sub_query = sub_query.subquery()
all_data = session.query(Trait)\
.join(sub_query, Trait.event_id == sub_query.c.id)
# Now convert the sqlalchemy objects back into Models ...
event_models_dict = {}
for trait in all_data.all():
event = event_models_dict.get(trait.event_id)
if not event:
generated = utils.decimal_to_dt(trait.event.generated)
event = api_models.Event(trait.event.unique_name.key,
generated, [])
event_models_dict[trait.event_id] = event
value = trait.get_value()
trait_model = api_models.Trait(trait.name.key, trait.t_type,
value)
event.append_trait(trait_model)
event_models = event_models_dict.values()
return sorted(event_models, key=operator.attrgetter('generated'))
def _make_stats_query(sample_filter):
session = sqlalchemy_session.get_session()
query = session.query(
func.min(Meter.timestamp).label('tsmin'),
func.max(Meter.timestamp).label('tsmax'),
func.avg(Meter.counter_volume).label('avg'),
func.sum(Meter.counter_volume).label('sum'),
func.min(Meter.counter_volume).label('min'),
func.max(Meter.counter_volume).label('max'),
func.count(Meter.counter_volume).label('count'))
return make_query_from_filter(query, sample_filter)
def record_alarm_change(self, alarm_change):
"""Record alarm change event.
"""
session = sqlalchemy_session.get_session()
with session.begin():
session.merge(User(id=alarm_change['user_id']))
session.merge(Project(id=alarm_change['project_id']))
session.merge(Project(id=alarm_change['on_behalf_of']))
alarm_change_row = AlarmChange(event_id=alarm_change['event_id'])
alarm_change_row.update(alarm_change)
session.add(alarm_change_row)
session.flush()
def _make_stats_query(sample_filter):
session = sqlalchemy_session.get_session()
query = session.query(
func.min(Meter.timestamp).label('tsmin'),
func.max(Meter.timestamp).label('tsmax'),
func.avg(Meter.counter_volume).label('avg'),
func.sum(Meter.counter_volume).label('sum'),
func.min(Meter.counter_volume).label('min'),
func.max(Meter.counter_volume).label('max'),
func.count(Meter.counter_volume).label('count'))
return make_query_from_filter(query, sample_filter)
def record_alarm_change(self, alarm_change):
"""Record alarm change event.
"""
session = sqlalchemy_session.get_session()
with session.begin():
session.merge(User(id=alarm_change['user_id']))
session.merge(Project(id=alarm_change['project_id']))
session.merge(Project(id=alarm_change['on_behalf_of']))
alarm_change_row = AlarmChange(event_id=alarm_change['event_id'])
alarm_change_row.update(alarm_change)
session.add(alarm_change_row)
session.flush()
def _real_record_metering_data(cls, data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.meter.meter_message_from_counter
"""
session = sqlalchemy_session.get_session()
with session.begin():
# Record the updated resource metadata
rmetadata = data['resource_metadata']
source = cls._create_or_update(session, models.Source,
data['source'])
user = cls._create_or_update(session, models.User, data['user_id'],
source)
project = cls._create_or_update(session, models.Project,
data['project_id'], source)
resource = cls._create_or_update(session,
models.Resource,
data['resource_id'],
source,
user=user,
project=project,
resource_metadata=rmetadata)
# Record the raw data for the meter.
meter = models.Meter(counter_type=data['counter_type'],
counter_unit=data['counter_unit'],
counter_name=data['counter_name'],
resource=resource)
session.add(meter)
if not filter(lambda x: x.id == source.id, meter.sources):
meter.sources.append(source)
meter.project = project
meter.user = user
meter.timestamp = data['timestamp']
meter.resource_metadata = rmetadata
meter.counter_volume = data['counter_volume']
meter.message_signature = data['message_signature']
meter.message_id = data['message_id']
session.flush()
if rmetadata:
if isinstance(rmetadata, dict):
for key, v in utils.dict_to_keyval(rmetadata):
try:
_model = META_TYPE_MAP[type(v)]
except KeyError:
LOG.warn(
_("Unknown metadata type. Key (%s) will "
"not be queryable."), key)
else:
session.add(
_model(id=meter.id, meta_key=key, value=v))
def update_alarm(self, alarm):
"""Update an alarm.
:param alarm: the new Alarm to update
"""
session = sqlalchemy_session.get_session()
with session.begin():
alarm_row = session.merge(Alarm(id=alarm.alarm_id))
alarm_row.update(alarm.as_dict())
session.flush()
return self._row_to_alarm_model(alarm_row)
def __init__(self, conf):
url = conf.database.connection
if url == 'sqlite://':
conf.database.connection = \
os.environ.get('CEILOMETER_TEST_SQL_URL', url)
session = sqlalchemy_session.get_session()
engine = session.get_bind()
if isinstance(engine.pool, pool.QueuePool):
poolsize = engine.pool.size() + engine.pool._max_overflow
self.pool = eventlet.GreenPool(poolsize)
else:
self.pool = None
def test_clear_metering_data_meta_tables(self):
timeutils.utcnow.override_time = datetime.datetime(2012, 7, 2, 10, 45)
self.conn.clear_expired_metering_data(3 * 60)
session = sqlalchemy_session.get_session()
meta_tables = [sql_models.MetaText, sql_models.MetaFloat,
sql_models.MetaBigInt, sql_models.MetaBool]
for table in meta_tables:
self.assertEqual(session.query(table)
.filter(~table.id.in_(
session.query(sql_models.Meter.id)
.group_by(sql_models.Meter.id)
)).count(), 0)
def _make_stats_query(sample_filter):
session = sqlalchemy_session.get_session()
query = session.query(
func.min(Meter.timestamp).label("tsmin"),
func.max(Meter.timestamp).label("tsmax"),
func.avg(Meter.counter_volume).label("avg"),
func.sum(Meter.counter_volume).label("sum"),
func.min(Meter.counter_volume).label("min"),
func.max(Meter.counter_volume).label("max"),
func.count(Meter.counter_volume).label("count"),
)
return make_query_from_filter(query, sample_filter)
def __init__(self, conf):
url = conf.database.connection
if url == 'sqlite://':
conf.database.connection = \
os.environ.get('CEILOMETER_TEST_SQL_URL', url)
session = sqlalchemy_session.get_session()
engine = session.get_bind()
if isinstance(engine.pool, pool.QueuePool):
poolsize = engine.pool.size() + engine.pool._max_overflow
self.pool = eventlet.GreenPool(poolsize)
else:
self.pool = None
def clear_expired_metering_data(ttl):
"""Clear expired data from the backend storage system according to the
time-to-live.
:param ttl: Number of seconds to keep records for.
"""
session = sqlalchemy_session.get_session()
with session.begin():
end = timeutils.utcnow() - datetime.timedelta(seconds=ttl)
meter_query = session.query(models.Meter)\
.filter(models.Meter.timestamp < end)
for meter_obj in meter_query.all():
session.delete(meter_obj)
query = session.query(models.User).filter(
~models.User.id.in_(
session.query(models.Meter.user_id).group_by(
models.Meter.user_id)), ~models.User.id.in_(
session.query(models.Alarm.user_id).group_by(
models.Alarm.user_id)),
~models.User.id.in_(
session.query(models.AlarmChange.user_id).group_by(
models.AlarmChange.user_id)))
for user_obj in query.all():
session.delete(user_obj)
query = session.query(models.Project)\
.filter(~models.Project.id.in_(
session.query(models.Meter.project_id)
.group_by(models.Meter.project_id)),
~models.Project.id.in_(
session.query(models.Alarm.project_id)
.group_by(models.Alarm.project_id)),
~models.Project.id.in_(
session.query(models.AlarmChange.project_id)
.group_by(models.AlarmChange.project_id)),
~models.Project.id.in_(
session.query(models.AlarmChange.on_behalf_of)
.group_by(models.AlarmChange.on_behalf_of))
)
for project_obj in query.all():
session.delete(project_obj)
query = session.query(models.Resource)\
.filter(~models.Resource.id.in_(
session.query(models.Meter.resource_id).group_by(
models.Meter.resource_id)))
for res_obj in query.all():
session.delete(res_obj)
def clear_expired_metering_data(ttl):
"""Clear expired data from the backend storage system according to the
time-to-live.
:param ttl: Number of seconds to keep records for.
"""
session = sqlalchemy_session.get_session()
with session.begin():
end = timeutils.utcnow() - datetime.timedelta(seconds=ttl)
meter_query = session.query(models.Meter)\
.filter(models.Meter.timestamp < end)
for meter_obj in meter_query.all():
session.delete(meter_obj)
query = session.query(models.User).filter(
~models.User.id.in_(session.query(models.Meter.user_id)
.group_by(models.Meter.user_id)),
~models.User.id.in_(session.query(models.Alarm.user_id)
.group_by(models.Alarm.user_id)),
~models.User.id.in_(session.query(models.AlarmChange.user_id)
.group_by(models.AlarmChange.user_id))
)
for user_obj in query.all():
session.delete(user_obj)
query = session.query(models.Project)\
.filter(~models.Project.id.in_(
session.query(models.Meter.project_id)
.group_by(models.Meter.project_id)),
~models.Project.id.in_(
session.query(models.Alarm.project_id)
.group_by(models.Alarm.project_id)),
~models.Project.id.in_(
session.query(models.AlarmChange.project_id)
.group_by(models.AlarmChange.project_id)),
~models.Project.id.in_(
session.query(models.AlarmChange.on_behalf_of)
.group_by(models.AlarmChange.on_behalf_of))
)
for project_obj in query.all():
session.delete(project_obj)
query = session.query(models.Resource)\
.filter(~models.Resource.id.in_(
session.query(models.Meter.resource_id).group_by(
models.Meter.resource_id)))
for res_obj in query.all():
session.delete(res_obj)
def _real_record_metering_data(cls, data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.meter.meter_message_from_counter
"""
session = sqlalchemy_session.get_session()
with session.begin():
# Record the updated resource metadata
rmetadata = data['resource_metadata']
source = cls._create_or_update(session, models.Source,
data['source'])
user = cls._create_or_update(session, models.User, data['user_id'],
source)
project = cls._create_or_update(session, models.Project,
data['project_id'], source)
resource = cls._create_or_update(session, models.Resource,
data['resource_id'], source,
user=user, project=project,
resource_metadata=rmetadata)
# Record the raw data for the meter.
meter = models.Meter(counter_type=data['counter_type'],
counter_unit=data['counter_unit'],
counter_name=data['counter_name'],
resource=resource)
session.add(meter)
if not filter(lambda x: x.id == source.id, meter.sources):
meter.sources.append(source)
meter.project = project
meter.user = user
meter.timestamp = data['timestamp']
meter.resource_metadata = rmetadata
meter.counter_volume = data['counter_volume']
meter.message_signature = data['message_signature']
meter.message_id = data['message_id']
session.flush()
if rmetadata:
if isinstance(rmetadata, dict):
for key, v in utils.dict_to_keyval(rmetadata):
try:
_model = META_TYPE_MAP[type(v)]
except KeyError:
LOG.warn(_("Unknown metadata type. Key (%s) will "
"not be queryable."), key)
else:
session.add(_model(id=meter.id,
meta_key=key,
value=v))
def test_clear_metering_data_meta_tables(self):
timeutils.utcnow.override_time = datetime.datetime(2012, 7, 2, 10, 45)
self.conn.clear_expired_metering_data(3 * 60)
session = sqlalchemy_session.get_session()
meta_tables = [
sql_models.MetaText, sql_models.MetaFloat, sql_models.MetaBigInt,
sql_models.MetaBool
]
for table in meta_tables:
self.assertEqual(
session.query(table).filter(~table.id.in_(
session.query(sql_models.Meter.id).group_by(
sql_models.Meter.id))).count(), 0)
def _get_or_create_trait_type(trait_type, data_type, session=None):
"""Find if this trait already exists in the database, and
if it does not, create a new entry in the trait type table.
"""
if session is None:
session = sqlalchemy_session.get_session()
with session.begin(subtransactions=True):
tt = session.query(models.TraitType).filter(
models.TraitType.desc == trait_type,
models.TraitType.data_type == data_type).first()
if not tt:
tt = models.TraitType(trait_type, data_type)
session.add(tt)
return tt
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = sqlalchemy_session.get_session().get_bind()
connection = engine.connect()
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def _get_or_create_trait_type(trait_type, data_type, session=None):
"""Find if this trait already exists in the database, and
if it does not, create a new entry in the trait type table.
"""
if session is None:
session = sqlalchemy_session.get_session()
with session.begin(subtransactions=True):
tt = session.query(models.TraitType).filter(
models.TraitType.desc == trait_type,
models.TraitType.data_type == data_type).first()
if not tt:
tt = models.TraitType(trait_type, data_type)
session.add(tt)
return tt
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = sqlalchemy_session.get_session().get_bind()
connection = engine.connect()
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def update_alarm(self, alarm):
"""Update an alarm.
:param alarm: the new Alarm to update
"""
session = sqlalchemy_session.get_session()
with session.begin():
Connection._create_or_update(session, models.User, alarm.user_id)
Connection._create_or_update(session, models.Project,
alarm.project_id)
alarm_row = session.merge(models.Alarm(id=alarm.alarm_id))
alarm_row.update(alarm.as_dict())
return self._row_to_alarm_model(alarm_row)
def _get_or_create_unique_name(self, key, session=None):
"""Find the UniqueName entry for a given key, creating
one if necessary.
This may result in a flush.
"""
if session is None:
session = sqlalchemy_session.get_session()
with session.begin(subtransactions=True):
unique = self._get_unique(session, key)
if not unique:
unique = UniqueName(key=key)
session.add(unique)
session.flush()
return unique
def create_alarm(self, alarm):
"""Create an alarm.
:param alarm: The alarm to create.
"""
session = sqlalchemy_session.get_session()
with session.begin():
session.merge(User(id=alarm.user_id))
session.merge(Project(id=alarm.project_id))
alarm_row = Alarm(id=alarm.alarm_id)
alarm_row.update(alarm.as_dict())
session.add(alarm_row)
session.flush()
return self._row_to_alarm_model(alarm_row)
def _get_or_create_unique_name(self, key, session=None):
"""Find the UniqueName entry for a given key, creating
one if necessary.
This may result in a flush.
"""
if session is None:
session = sqlalchemy_session.get_session()
with session.begin(subtransactions=True):
unique = self._get_unique(session, key)
if not unique:
unique = UniqueName(key=key)
session.add(unique)
session.flush()
return unique
def create_alarm(self, alarm):
"""Create an alarm.
:param alarm: The alarm to create.
"""
session = sqlalchemy_session.get_session()
with session.begin():
session.merge(User(id=alarm.user_id))
session.merge(Project(id=alarm.project_id))
alarm_row = Alarm(id=alarm.alarm_id)
alarm_row.update(alarm.as_dict())
session.add(alarm_row)
session.flush()
return self._row_to_alarm_model(alarm_row)
def save(self, session=None):
"""Save this object."""
if not session:
session = sa.get_session()
# NOTE(boris-42): This part of code should be look like:
# session.add(self)
# session.flush()
# But there is a bug in sqlalchemy and eventlet that
# raises NoneType exception if there is no running
# transaction and rollback is called. As long as
# sqlalchemy has this bug we have to create transaction
# explicitly.
with session.begin(subtransactions=True):
session.add(self)
session.flush()
def _get_or_create_event_type(event_type, session=None):
"""Here, we check to see if an event type with the supplied
name already exists. If not, we create it and return the record.
This may result in a flush.
"""
if session is None:
session = sqlalchemy_session.get_session()
with session.begin(subtransactions=True):
et = session.query(models.EventType).filter(
models.EventType.desc == event_type).first()
if not et:
et = models.EventType(event_type)
session.add(et)
return et
def record_alarm_change(self, alarm_change):
"""Record alarm change event.
"""
session = sqlalchemy_session.get_session()
with session.begin():
Connection._create_or_update(session, models.User,
alarm_change['user_id'])
Connection._create_or_update(session, models.Project,
alarm_change['project_id'])
Connection._create_or_update(session, models.Project,
alarm_change['on_behalf_of'])
alarm_change_row = models.AlarmChange(
event_id=alarm_change['event_id'])
alarm_change_row.update(alarm_change)
session.add(alarm_change_row)
def _get_or_create_event_type(event_type, session=None):
"""Here, we check to see if an event type with the supplied
name already exists. If not, we create it and return the record.
This may result in a flush.
"""
if session is None:
session = sqlalchemy_session.get_session()
with session.begin(subtransactions=True):
et = session.query(models.EventType).filter(
models.EventType.desc == event_type).first()
if not et:
et = models.EventType(event_type)
session.add(et)
return et
