def get_beam_query_from_ndb_query(query):
"""Returns an equivalent Apache Beam query from the given NDB query.
This function helps developers avoid learning two types of query syntaxes.
Specifically, the datastoreio module offered by the Apache Beam SDK only
accepts Beam datastore queries, and are implemented very differently from
NDB queries. This function adapts the two patterns to make job code easier
to write.
Args:
query: datastore_services.Query. The NDB query to convert.
Returns:
beam_datastore_types.Query. The equivalent Apache Beam query.
"""
kind = query.kind
namespace = query.namespace
project = query.app
if query.filters:
filters = _get_beam_filters_from_ndb_filter_node(query.filters)
else:
filters = None
if query.orders:
order = _get_beam_order_from_ndb_order(query.orders)
else:
order = None
if kind is None and order is None:
order = ('__key__',)
return beam_datastore_types.Query(
kind=kind, namespace=namespace, project=project, filters=filters,
order=order)
def make_query(self,
kind=None,
namespace=None,
project=None,
filters=None,
order=None,
limit=None):
"""Returns a new beam_datastore_types.Query object.
Args:
kind: str|None. The kind to query. If None, all kinds are eligible.
namespace: str|None. Namespace to restrict results to.
project: str|None. Project associated with query.
filters: list(tuple(str,str,str))|None. Property filters applied
by this query. The sequence is:
`(property_name, operator, value)`.
order: list(str)|None. Field names used to order query results.
Prepend `-` to a field name to sort it in descending order.
limit: int|None. Maximum amount of results to return.
Returns:
beam_datastore_types.Query. The Query object.
"""
if kind is None and order is None:
order = ('__key__', )
return beam_datastore_types.Query(kind=kind,
namespace=namespace,
project=project,
filters=filters,
order=order,
limit=limit)
def test_create_scatter_query(self):
query = types.Query(kind='shakespeare-demo')
num_splits = 10
scatter_query = query_splitter._create_scatter_query(query, num_splits)
self.assertEqual(scatter_query.kind, query.kind)
self.assertEqual(scatter_query.limit,
(num_splits - 1) * query_splitter.KEYS_PER_SPLIT)
self.assertEqual(scatter_query.order,
[query_splitter.SCATTER_PROPERTY_NAME])
self.assertEqual(scatter_query.projection,
[query_splitter.KEY_PROPERTY_NAME])
def _create_scatter_query(query, num_splits):
"""Creates a scatter query from the given user query."""
# There is a split containing entities before and after each scatter entity:
# ||---*------*------*------*------*------*------*---|| * = scatter entity
# If we represent each split as a region before a scatter entity, there is an
# extra region following the last scatter point. Thus, we do not need the
# scatter entity for the last region.
limit = (num_splits - 1) * KEYS_PER_SPLIT
scatter_query = types.Query(
kind=query.kind, project=query.project, namespace=query.namespace,
order=[SCATTER_PROPERTY_NAME],
projection=[KEY_PROPERTY_NAME], limit=limit)
return scatter_query
def create_query(self, kinds=(), order=False, limit=None, offset=None,
inequality_filter=False):
if len(kinds) > 1:
self.skipTest('v1new queries do not support more than one kind.')
if offset is not None:
self.skipTest('v1new queries do not support offsets.')
kind = None
filters = []
if kinds:
kind = kinds[0]
if order:
order = ['prop1']
if inequality_filter:
filters = [('prop1', '>', 'value1')]
return types.Query(kind=kind, filters=filters, order=order, limit=limit)
def get_beam_query_from_ndb_query(query, namespace=None):
"""Returns an equivalent Apache Beam query from the given NDB query.
This function helps developers avoid learning two types of query syntaxes.
Specifically, the datastoreio module offered by the Apache Beam SDK only
accepts Beam datastore queries, and are implemented very differently from
NDB queries. This function adapts the two patterns to make job code easier
to write.
Args:
query: datastore_services.Query. The NDB query to convert.
namespace: str|None. Namespace for isolating the NDB operations of
tests.
Returns:
beam_datastore_types.Query. The equivalent Apache Beam query.
"""
kind = query.kind
# This is needed mainly for testing and it adds an easy way to force
# the queries into their own namespace.
namespace = namespace or query.namespace
if query.filters:
filters = _get_beam_filters_from_ndb_filter_node(query.filters)
else:
filters = None
if query.order_by:
order = _get_beam_order_from_ndb_order(query.order_by)
else:
order = ()
if kind is None and not order:
order = ('__key__',)
return beam_datastore_types.Query(
kind=kind,
namespace=namespace,
project=feconf.OPPIA_PROJECT_ID,
filters=filters,
order=order
)
def get_beam_query_from_ndb_query(
query: datastore_services.Query,
namespace: Optional[str] = None) -> beam_datastore_types.Query:
"""Returns an equivalent Apache Beam query from the given NDB query.
This function helps developers avoid learning two types of query syntaxes.
Specifically, the datastoreio module offered by the Apache Beam SDK only
accepts Beam datastore queries, and are implemented very differently from
NDB queries. This function adapts the two patterns to make job code easier
to write.
Args:
query: datastore_services.Query. The NDB query to convert.
namespace: str|None. Namespace for isolating the NDB operations of unit
tests. IMPORTANT: Do not use this argument outside of unit tests.
Returns:
beam_datastore_types.Query. The equivalent Apache Beam query.
"""
kind = query.kind
namespace = namespace or query.namespace
filters = (_get_beam_filters_from_ndb_node(query.filters)
if query.filters else ())
order = (_get_beam_order_from_ndb_order(query.order_by)
if query.order_by else ())
if not kind and not order:
# NOTE: When kind is omitted, Apache Beam requires the query to order
# its results by __key__ (the models' .key() value).
order = ('__key__', )
return beam_datastore_types.Query(kind=kind,
namespace=namespace,
project=feconf.OPPIA_PROJECT_ID,
filters=filters,
order=order)
