def network_get_associated_fixed_ips(context, network_id, host=None):
# FIXME(sirp): since this returns fixed_ips, this would be better named
# fixed_ip_get_all_by_network.
# NOTE(vish): The ugly joins here are to solve a performance issue and
# should be removed once we can add and remove leases
# without regenerating the whole list
vif_and = and_(models.VirtualInterface.id ==
models.FixedIp.virtual_interface_id,
models.VirtualInterface.deleted == 1)
inst_and = and_(models.Instance.uuid == models.FixedIp.instance_uuid,
models.Instance.deleted == 1)
session = get_session()
query = session.query(models.FixedIp.address,
models.FixedIp.instance_uuid,
models.FixedIp.network_id,
models.FixedIp.virtual_interface_id,
models.VirtualInterface.address,
models.Instance.hostname,
models.Instance.updated_at,
models.Instance.created_at,
models.FixedIp.allocated,
models.FixedIp.leased)
query = query.join(models.VirtualInterface).join(models.Instance)
query = query.filter(models.FixedIp.deleted == 0)
query = query.filter(models.FixedIp.network_id == network_id)
query = query.join((models.VirtualInterface, vif_and))
query = query.filter(models.FixedIp.instance_uuid != None)
query = query.filter(models.FixedIp.virtual_interface_id != None)
# query = query.filter(models.FixedIp.deleted == 0).\
# filter(models.FixedIp.network_id == network_id).\
# join((models.VirtualInterface, vif_and)).\
# join((models.Instance, inst_and)).\
# filter(models.FixedIp.instance_uuid != None).\
# filter(models.FixedIp.virtual_interface_id != None)
if host:
query = query.filter(models.Instance.host == host)
result = query.all()
plop1 = Query(models.FixedIp).join(models.Instance).filter(models.Instance.uuid==models.FixedIp.instance_uuid).all()
print(plop1)
data = []
for datum in result:
cleaned = {}
cleaned['address'] = datum[0]
cleaned['instance_uuid'] = datum[1]
cleaned['network_id'] = datum[2]
cleaned['vif_id'] = datum[3]
cleaned['vif_address'] = datum[4]
cleaned['instance_hostname'] = datum[5]
cleaned['instance_updated'] = datum[6]
cleaned['instance_created'] = datum[7]
cleaned['allocated'] = datum[8]
cleaned['leased'] = datum[9]
cleaned['default_route'] = datum[10] is not None
data.append(cleaned)
return data
def network_get_associated_fixed_ips(context, network_id, host=None):
# FIXME(sirp): since this returns fixed_ips, this would be better named
# fixed_ip_get_all_by_network.
# NOTE(vish): The ugly joins here are to solve a performance issue and
# should be removed once we can add and remove leases
# without regenerating the whole list
vif_and = and_(
models.VirtualInterface.id == models.FixedIp.virtual_interface_id,
models.VirtualInterface.deleted == 1)
inst_and = and_(models.Instance.uuid == models.FixedIp.instance_uuid,
models.Instance.deleted == 1)
session = get_session()
query = session.query(models.FixedIp.address, models.FixedIp.instance_uuid,
models.FixedIp.network_id,
models.FixedIp.virtual_interface_id,
models.VirtualInterface.address,
models.Instance.hostname, models.Instance.updated_at,
models.Instance.created_at, models.FixedIp.allocated,
models.FixedIp.leased)
query = query.join(models.VirtualInterface).join(models.Instance)
query = query.filter(models.FixedIp.deleted == 0)
query = query.filter(models.FixedIp.network_id == network_id)
query = query.join((models.VirtualInterface, vif_and))
query = query.filter(models.FixedIp.instance_uuid != None)
query = query.filter(models.FixedIp.virtual_interface_id != None)
# query = query.filter(models.FixedIp.deleted == 0).\
# filter(models.FixedIp.network_id == network_id).\
# join((models.VirtualInterface, vif_and)).\
# join((models.Instance, inst_and)).\
# filter(models.FixedIp.instance_uuid != None).\
# filter(models.FixedIp.virtual_interface_id != None)
if host:
query = query.filter(models.Instance.host == host)
result = query.all()
plop1 = Query(models.FixedIp).join(models.Instance).filter(
models.Instance.uuid == models.FixedIp.instance_uuid).all()
print(plop1)
data = []
for datum in result:
cleaned = {}
cleaned['address'] = datum[0]
cleaned['instance_uuid'] = datum[1]
cleaned['network_id'] = datum[2]
cleaned['vif_id'] = datum[3]
cleaned['vif_address'] = datum[4]
cleaned['instance_hostname'] = datum[5]
cleaned['instance_updated'] = datum[6]
cleaned['instance_created'] = datum[7]
cleaned['allocated'] = datum[8]
cleaned['leased'] = datum[9]
cleaned['default_route'] = datum[10] is not None
data.append(cleaned)
return data
def migration_get_in_progress_by_host_and_node(context, host, node):
return model_query(context, models.Migration).\
filter(or_(and_(models.Migration.source_compute == host,
models.Migration.source_node == node),
and_(models.Migration.dest_compute == host,
models.Migration.dest_node == node))).\
filter(~models.Migration.status.in_(['confirmed', 'reverted',
'error'])).\
all()
def test_expression_and_or(self):
"""Checks on a specified attribute with operators "IN"."""
# Checks several examples with "and" and "or" operators
expression = BooleanExpression("NORMAL", or_(and_(models.Network.label != "network_3", models.Network.multi_host == True), models.Network.label == "network_3"))
value = expression.evaluate(KeyedTuple([{"label": "network_3", "multi_host": False}], ["networks"]))
self.assertTrue(value, "complex expression (1)")
expression = BooleanExpression("NORMAL", or_(and_(models.Network.label != "network_3", models.Network.multi_host == True), models.Network.label == "network_3"))
value = expression.evaluate(KeyedTuple([{"label": "network_2", "multi_host": True}], ["networks"]))
self.assertTrue(value, "complex expression (2)")
expression = BooleanExpression("NORMAL", or_(and_(models.Network.label != "network_3", models.Network.multi_host == True), models.Network.label == "network_3"))
value = expression.evaluate(KeyedTuple([{"label": "network_2", "multi_host": False}], ["networks"]))
self.assertFalse(value, "complex expression (3)")
def _select_images_query(context, image_conditions, admin_as_user,
member_status, visibility):
session = get_session()
img_conditional_clause = and_(*image_conditions)
regular_user = (not context.is_admin) or admin_as_user
query_member = session.query(models.Image).join(
models.ImageMember).filter(img_conditional_clause)
if regular_user:
member_filters = [models.ImageMember.deleted == False]
if context.owner is not None:
member_filters.extend([models.ImageMember.member == context.owner])
if member_status != 'all':
member_filters.extend(
[models.ImageMember.status == member_status])
query_member = query_member.filter(and_(*member_filters))
# NOTE(venkatesh) if the 'visibility' is set to 'shared', we just
# query the image members table. No union is required.
if visibility is not None and visibility == 'shared':
return query_member
query_image = session.query(models.Image).filter(img_conditional_clause)
if regular_user:
query_image = query_image.filter(models.Image.is_public == True)
query_image_owner = None
if context.owner is not None:
query_image_owner = session.query(models.Image).filter(
models.Image.owner == context.owner).filter(
img_conditional_clause)
if query_image_owner is not None:
query = query_image.union(query_image_owner, query_member)
else:
query = query_image.union(query_member)
return query
else:
# Admin user
return query_image
def _select_images_query(context, image_conditions, admin_as_user,
member_status, visibility):
session = get_session()
img_conditional_clause = and_(*image_conditions)
regular_user = (not context.is_admin) or admin_as_user
query_member = session.query(models.Image).join(
models.ImageMember).filter(img_conditional_clause)
if regular_user:
member_filters = [models.ImageMember.deleted == False]
if context.owner is not None:
member_filters.extend([models.ImageMember.member == context.owner])
if member_status != 'all':
member_filters.extend([
models.ImageMember.status == member_status])
query_member = query_member.filter(and_(*member_filters))
# NOTE(venkatesh) if the 'visibility' is set to 'shared', we just
# query the image members table. No union is required.
if visibility is not None and visibility == 'shared':
return query_member
query_image = session.query(models.Image).filter(img_conditional_clause)
if regular_user:
query_image = query_image.filter(models.Image.is_public == True)
query_image_owner = None
if context.owner is not None:
query_image_owner = session.query(models.Image).filter(
models.Image.owner == context.owner).filter(
img_conditional_clause)
if query_image_owner is not None:
query = query_image.union(query_image_owner, query_member)
else:
query = query_image.union(query_member)
return query
else:
# Admin user
return query_image
def network_get_all_by_host(host):
fixed_host_filter = or_(
models.FixedIp.host == host, and_(models.FixedIp.instance_uuid != None, models.Instance.host == host)
)
fixed_ip_query = (
Query(models.FixedIp.network_id)
.outerjoin((models.Instance, models.Instance.uuid == models.FixedIp.instance_uuid))
.filter(fixed_host_filter)
)
# NOTE(vish): return networks that have host set
# or that have a fixed ip with host set
# or that have an instance with host set
host_filter = or_(models.Network.host == host, models.Network.id.in_(fixed_ip_query.subquery()))
return _network_get_query().filter(host_filter).all()
def fixed_ip_get_by_instance(context, instance_uuid):
if not uuidutils.is_uuid_like(instance_uuid):
raise Exception("invalid UUID")
vif_and = and_(models.VirtualInterface.id ==
models.FixedIp.virtual_interface_id,
models.VirtualInterface.deleted == 0)
result = model_query(context, models.FixedIp, read_deleted="no").\
filter(models.FixedIp.instance_uuid==instance_uuid).\
outerjoin(models.VirtualInterface, vif_and).\
all()
if not result:
raise Exception("FixedIpNotFoundForInstance(instance_uuid=%s)" % (instance_uuid))
# TODO(Jonathan): quick fix
return [x[0] for x in result]
def fixed_ip_get_by_instance(context, instance_uuid):
if not uuidutils.is_uuid_like(instance_uuid):
raise Exception("invalid UUID")
vif_and = and_(
models.VirtualInterface.id == models.FixedIp.virtual_interface_id,
models.VirtualInterface.deleted == 0)
result = model_query(context, models.FixedIp, read_deleted="no").\
filter(models.FixedIp.instance_uuid==instance_uuid).\
outerjoin(models.VirtualInterface, vif_and).\
all()
if not result:
raise Exception("FixedIpNotFoundForInstance(instance_uuid=%s)" %
(instance_uuid))
# TODO(Jonathan): quick fix
return [x[0] for x in result]
def image_get_all(context,
filters=None,
marker=None,
limit=None,
sort_key=None,
sort_dir=None,
member_status='accepted',
is_public=None,
admin_as_user=False,
return_tag=False):
"""
Get all images that match zero or more filters.
:param filters: dict of filter keys and values. If a 'properties'
key is present, it is treated as a dict of key/value
filters on the image properties attribute
:param marker: image id after which to start page
:param limit: maximum number of images to return
:param sort_key: list of image attributes by which results should be sorted
:param sort_dir: directions in which results should be sorted (asc, desc)
:param member_status: only return shared images that have this membership
status
:param is_public: If true, return only public images. If false, return
only private and shared images.
:param admin_as_user: For backwards compatibility. If true, then return to
an admin the equivalent set of images which it would see
if it was a regular user
:param return_tag: To indicates whether image entry in result includes it
relevant tag entries. This could improve upper-layer
query performance, to prevent using separated calls
"""
sort_key = ['created_at'] if not sort_key else sort_key
default_sort_dir = 'desc'
if not sort_dir:
sort_dir = [default_sort_dir] * len(sort_key)
elif len(sort_dir) == 1:
default_sort_dir = sort_dir[0]
sort_dir *= len(sort_key)
filters = filters or {}
visibility = filters.pop('visibility', None)
showing_deleted = 'changes-since' in filters or filters.get(
'deleted', False)
img_cond, prop_cond, tag_cond = _make_conditions_from_filters(
filters, is_public)
query = _select_images_query(context, img_cond, admin_as_user,
member_status, visibility)
if visibility is not None:
if visibility == 'public':
query = query.filter(models.Image.is_public == True)
elif visibility == 'private':
query = query.filter(models.Image.is_public == False)
if prop_cond:
for prop_condition in prop_cond:
query = query.join(models.ImageProperty,
aliased=True).filter(and_(*prop_condition))
if tag_cond:
for tag_condition in tag_cond:
query = query.join(models.ImageTag,
aliased=True).filter(and_(*tag_condition))
marker_image = None
if marker is not None:
marker_image = _image_get(context,
marker,
force_show_deleted=showing_deleted)
for key in ['created_at', 'id']:
if key not in sort_key:
sort_key.append(key)
sort_dir.append(default_sort_dir)
query = _paginate_query(query,
models.Image,
limit,
sort_key,
marker=marker_image,
sort_dir=None,
sort_dirs=sort_dir)
query = query.options(sa_orm.joinedload(models.Image.properties)).options(
sa_orm.joinedload(models.Image.locations))
if return_tag:
query = query.options(sa_orm.joinedload(models.Image.tags))
images = []
for image in query.all():
img = image[0] if type(image) is list else image
image_dict = img.to_dict()
image_dict = _normalize_locations(context,
image_dict,
force_show_deleted=showing_deleted)
if return_tag:
image_dict = _normalize_tags(image_dict)
images.append(image_dict)
return images
def _paginate_query(query,
model,
limit,
sort_keys,
marker=None,
sort_dir=None,
sort_dirs=None):
"""Returns a query with sorting / pagination criteria added.
Pagination works by requiring a unique sort_key, specified by sort_keys.
(If sort_keys is not unique, then we risk looping through values.)
We use the last row in the previous page as the 'marker' for pagination.
So we must return values that follow the passed marker in the order.
With a single-valued sort_key, this would be easy: sort_key > X.
With a compound-values sort_key, (k1, k2, k3) we must do this to repeat
the lexicographical ordering:
(k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3)
We also have to cope with different sort_directions.
Typically, the id of the last row is used as the client-facing pagination
marker, then the actual marker object must be fetched from the db and
passed in to us as marker.
:param query: the query object to which we should add paging/sorting
:param model: the ORM model class
:param limit: maximum number of items to return
:param sort_keys: array of attributes by which results should be sorted
:param marker: the last item of the previous page; we returns the next
results after this value.
:param sort_dir: direction in which results should be sorted (asc, desc)
:param sort_dirs: per-column array of sort_dirs, corresponding to sort_keys
:rtype: sqlalchemy.orm.query.Query
:return: The query with sorting/pagination added.
"""
if 'id' not in sort_keys:
# TODO(justinsb): If this ever gives a false-positive, check
# the actual primary key, rather than assuming its id
LOG.warn('Id not in sort_keys; is sort_keys unique?')
assert (not (sort_dir and sort_dirs))
# Default the sort direction to ascending
if sort_dirs is None and sort_dir is None:
sort_dir = 'asc'
# Ensure a per-column sort direction
if sort_dirs is None:
sort_dirs = [sort_dir for _sort_key in sort_keys]
assert (len(sort_dirs) == len(sort_keys))
# Add sorting
for current_sort_key, current_sort_dir in zip(sort_keys, sort_dirs):
sort_dir_func = {
'asc': sqlalchemy.asc,
'desc': sqlalchemy.desc,
}[current_sort_dir]
try:
sort_key_attr = getattr(model, current_sort_key)
except AttributeError:
raise Exception("exception.InvalidSortKey()")
query = query.order_by(sort_dir_func(sort_key_attr))
default = '' # Default to an empty string if NULL
# Add pagination
if marker is not None:
marker_values = []
for sort_key in sort_keys:
v = getattr(marker, sort_key)
if v is None:
v = default
marker_values.append(v)
# Build up an array of sort criteria as in the docstring
criteria_list = []
for i in range(len(sort_keys)):
crit_attrs = []
for j in range(i):
model_attr = getattr(model, sort_keys[j])
default = None if isinstance(
model_attr.property.columns[0].type,
sqlalchemy.DateTime) else ''
attr = sa_sql.expression.case([
(model_attr != None, model_attr),
],
else_=default)
crit_attrs.append((attr == marker_values[j]))
model_attr = getattr(model, sort_keys[i])
default = None if isinstance(model_attr.property.columns[0].type,
sqlalchemy.DateTime) else ''
attr = sa_sql.expression.case([
(model_attr != None, model_attr),
],
else_=default)
if sort_dirs[i] == 'desc':
crit_attrs.append((attr < marker_values[i]))
elif sort_dirs[i] == 'asc':
crit_attrs.append((attr > marker_values[i]))
else:
raise ValueError("Unknown sort direction, "
"must be 'desc' or 'asc'")
criteria = and_(*crit_attrs)
criteria_list.append(criteria)
f = or_(*criteria_list)
query = query.filter(f)
if limit is not None:
query = query.limit(limit)
return query
def image_get_all(context, filters=None, marker=None, limit=None,
sort_key=None, sort_dir=None,
member_status='accepted', is_public=None,
admin_as_user=False, return_tag=False):
"""
Get all images that match zero or more filters.
:param filters: dict of filter keys and values. If a 'properties'
key is present, it is treated as a dict of key/value
filters on the image properties attribute
:param marker: image id after which to start page
:param limit: maximum number of images to return
:param sort_key: list of image attributes by which results should be sorted
:param sort_dir: directions in which results should be sorted (asc, desc)
:param member_status: only return shared images that have this membership
status
:param is_public: If true, return only public images. If false, return
only private and shared images.
:param admin_as_user: For backwards compatibility. If true, then return to
an admin the equivalent set of images which it would see
if it was a regular user
:param return_tag: To indicates whether image entry in result includes it
relevant tag entries. This could improve upper-layer
query performance, to prevent using separated calls
"""
sort_key = ['created_at'] if not sort_key else sort_key
default_sort_dir = 'desc'
if not sort_dir:
sort_dir = [default_sort_dir] * len(sort_key)
elif len(sort_dir) == 1:
default_sort_dir = sort_dir[0]
sort_dir *= len(sort_key)
filters = filters or {}
visibility = filters.pop('visibility', None)
showing_deleted = 'changes-since' in filters or filters.get('deleted',
False)
img_cond, prop_cond, tag_cond = _make_conditions_from_filters(
filters, is_public)
query = _select_images_query(context,
img_cond,
admin_as_user,
member_status,
visibility)
if visibility is not None:
if visibility == 'public':
query = query.filter(models.Image.is_public == True)
elif visibility == 'private':
query = query.filter(models.Image.is_public == False)
if prop_cond:
for prop_condition in prop_cond:
query = query.join(models.ImageProperty, aliased=True).filter(
and_(*prop_condition))
if tag_cond:
for tag_condition in tag_cond:
query = query.join(models.ImageTag, aliased=True).filter(
and_(*tag_condition))
marker_image = None
if marker is not None:
marker_image = _image_get(context,
marker,
force_show_deleted=showing_deleted)
for key in ['created_at', 'id']:
if key not in sort_key:
sort_key.append(key)
sort_dir.append(default_sort_dir)
query = _paginate_query(query, models.Image, limit,
sort_key,
marker=marker_image,
sort_dir=None,
sort_dirs=sort_dir)
query = query.options(sa_orm.joinedload(
models.Image.properties)).options(
sa_orm.joinedload(models.Image.locations))
if return_tag:
query = query.options(sa_orm.joinedload(models.Image.tags))
images = []
for image in query.all():
img = image[0] if type(image) is list else image
image_dict = img.to_dict()
image_dict = _normalize_locations(context, image_dict,
force_show_deleted=showing_deleted)
if return_tag:
image_dict = _normalize_tags(image_dict)
images.append(image_dict)
return images
def _paginate_query(query, model, limit, sort_keys, marker=None,
sort_dir=None, sort_dirs=None):
"""Returns a query with sorting / pagination criteria added.
Pagination works by requiring a unique sort_key, specified by sort_keys.
(If sort_keys is not unique, then we risk looping through values.)
We use the last row in the previous page as the 'marker' for pagination.
So we must return values that follow the passed marker in the order.
With a single-valued sort_key, this would be easy: sort_key > X.
With a compound-values sort_key, (k1, k2, k3) we must do this to repeat
the lexicographical ordering:
(k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3)
We also have to cope with different sort_directions.
Typically, the id of the last row is used as the client-facing pagination
marker, then the actual marker object must be fetched from the db and
passed in to us as marker.
:param query: the query object to which we should add paging/sorting
:param model: the ORM model class
:param limit: maximum number of items to return
:param sort_keys: array of attributes by which results should be sorted
:param marker: the last item of the previous page; we returns the next
results after this value.
:param sort_dir: direction in which results should be sorted (asc, desc)
:param sort_dirs: per-column array of sort_dirs, corresponding to sort_keys
:rtype: sqlalchemy.orm.query.Query
:return: The query with sorting/pagination added.
"""
if 'id' not in sort_keys:
# TODO(justinsb): If this ever gives a false-positive, check
# the actual primary key, rather than assuming its id
LOG.warn('Id not in sort_keys; is sort_keys unique?')
assert(not (sort_dir and sort_dirs))
# Default the sort direction to ascending
if sort_dirs is None and sort_dir is None:
sort_dir = 'asc'
# Ensure a per-column sort direction
if sort_dirs is None:
sort_dirs = [sort_dir for _sort_key in sort_keys]
assert(len(sort_dirs) == len(sort_keys))
# Add sorting
for current_sort_key, current_sort_dir in zip(sort_keys, sort_dirs):
sort_dir_func = {
'asc': sqlalchemy.asc,
'desc': sqlalchemy.desc,
}[current_sort_dir]
try:
sort_key_attr = getattr(model, current_sort_key)
except AttributeError:
raise Exception("exception.InvalidSortKey()")
query = query.order_by(sort_dir_func(sort_key_attr))
default = '' # Default to an empty string if NULL
# Add pagination
if marker is not None:
marker_values = []
for sort_key in sort_keys:
v = getattr(marker, sort_key)
if v is None:
v = default
marker_values.append(v)
# Build up an array of sort criteria as in the docstring
criteria_list = []
for i in range(len(sort_keys)):
crit_attrs = []
for j in range(i):
model_attr = getattr(model, sort_keys[j])
default = None if isinstance(
model_attr.property.columns[0].type,
sqlalchemy.DateTime) else ''
attr = sa_sql.expression.case([(model_attr != None,
model_attr), ],
else_=default)
crit_attrs.append((attr == marker_values[j]))
model_attr = getattr(model, sort_keys[i])
default = None if isinstance(model_attr.property.columns[0].type,
sqlalchemy.DateTime) else ''
attr = sa_sql.expression.case([(model_attr != None,
model_attr), ],
else_=default)
if sort_dirs[i] == 'desc':
crit_attrs.append((attr < marker_values[i]))
elif sort_dirs[i] == 'asc':
crit_attrs.append((attr > marker_values[i]))
else:
raise ValueError("Unknown sort direction, "
"must be 'desc' or 'asc'")
criteria = and_(*crit_attrs)
criteria_list.append(criteria)
f = or_(*criteria_list)
query = query.filter(f)
if limit is not None:
query = query.limit(limit)
return query
