def filter_ndb_query(self, query, filters=None):
"""Add query filter to restrict to this key range.
Args:
query: An ndb.Query instance.
filters: optional list of filters to apply to the query. Each filter is
a tuple: (<property_name_as_str>, <query_operation_as_str>, <value>).
User filters are applied first.
Returns:
The input query restricted to this key range.
"""
assert _IsNdbQuery(query)
if filters:
for f in filters:
query = query.filter(ndb.FilterNode(*f))
if self.include_start:
start_comparator = ">="
else:
start_comparator = ">"
if self.include_end:
end_comparator = "<="
else:
end_comparator = "<"
if self.key_start:
query = query.filter(
ndb.FilterNode("__key__", start_comparator, self.key_start))
if self.key_end:
query = query.filter(
ndb.FilterNode("__key__", end_comparator, self.key_end))
return query
def filter_ndb_query(self, query):
"""Add query filter to restrict to this key range.
Args:
query: An ndb.Query instance.
Returns:
The input query restricted to this key range.
"""
assert _IsNdbQuery(query)
if self.include_start:
start_comparator = ">="
else:
start_comparator = ">"
if self.include_end:
end_comparator = "<="
else:
end_comparator = "<"
if self.key_start:
query = query.filter(
ndb.FilterNode("__key__", start_comparator, self.key_start))
if self.key_end:
query = query.filter(
ndb.FilterNode("__key__", end_comparator, self.key_end))
return query
def _QueryModel(self, search_dict, ancestor=None):
"""Queries the model class for field-value pairs.
Args:
search_dict: A dictionary mapping from field name to search by to the
search term.
ancestor: ndb.Key, If provided, the ancestor for the query.
Returns:
The model query.
Raises:
QueryError: If the queried field is not a property of the model.
QueryTypeError: If search_term does not match the type of the search_base
model property.
"""
filter_nodes = []
for search_base, search_term in search_dict.items():
field_name = upvote_utils.CamelToSnakeCase(search_base)
# If the model class offers a translation function for property queries,
# invoke it and set the field and search term to the result.
try:
field_name, search_term = self.MODEL_CLASS.TranslatePropertyQuery(
field_name, search_term)
except AttributeError:
pass
else:
logging.info('Converted query to (%s = %s)', field_name,
search_term)
# Check for the property on the model itself (as opposed to, say, catching
# a getattr exception) to ensure that the field being accessed is an ndb
# property as opposed to a Python attribute.
if not model_utils.HasProperty(self.MODEL_CLASS, field_name):
raise QueryError('Invalid searchBase %s' % field_name)
field = getattr(self.MODEL_CLASS, field_name)
# If the field is of a non-string type, attempt to coerce the argument to
# conform to this type
search_term = _CoerceQueryParam(field, search_term)
filter_nodes.append(ndb.FilterNode(field_name, '=', search_term))
query = self.MODEL_CLASS.query(ancestor=ancestor)
if filter_nodes:
query = query.filter(ndb.AND(*filter_nodes))
return query
def make_query(self, ns):
"""Make a query of entities within this range.
Query options are not supported. They should be specified when the query
is run.
Args:
ns: namespace of this query.
Returns:
a db.Query or ndb.Query, depends on the model class's type.
"""
if issubclass(self.model_class, db.Model):
query = db.Query(self.model_class, namespace=ns)
for f in self.filters:
query.filter("%s %s" % (f[0], f[1]), f[2])
else:
query = self.model_class.query(namespace=ns)
for f in self.filters:
query = query.filter(ndb.FilterNode(*f))
return query
def get_list_for_model(args, model, or_filters=None, and_filters=None):
"""
Creates a list for the model
:param args:
:param model:
:param default_order:
:param or_filters:
:return: items, next_cursor, more, total_count_future
"""
default_order = -model.modified
if args.get('order'):
order_prop = model._properties.get(args.get('order'))
if order_prop:
default_order = -order_prop if args.get('order_reverse',
False) else order_prop
filter_node = None
or_filters = or_filters or []
if not isinstance(or_filters, list):
or_filters = [or_filters]
if args.get('filter'):
for filter in args.get('filter'):
# split filter into its components: e.g. filter=course,filter=2017,timestamp>12241231
_and_filters = and_filters or []
for item in filter.split('+'):
comparison = None
result = []
for delimiter in [
COMP_EQUALITY, COMP_GREATER_THAN, COMP_LESS_THAN
]:
result = [s.strip() for s in re.split(delimiter, item)]
if len(result) == 2:
comparison = delimiter
break
if comparison and result[0] in model._properties:
property = model.get_all_properties().get(result[0])
value = result[1]
if property and property.get('converter', None):
value = property.get('converter')(result[1])
_and_filters.append(
ndb.FilterNode(result[0], comparison, value))
if comparison != COMP_EQUALITY: # sort order must be altered
args['order'] = getattr(model, result[0])
if len(_and_filters) > 0:
or_filters.append(ndb.AND(*_and_filters))
elif and_filters:
or_filters.append(ndb.AND(*and_filters))
if len(or_filters) == 1:
filter_node = or_filters[0]
elif len(or_filters) > 0:
filter_node = ndb.OR(*or_filters)
items_future = model.query(filters=filter_node).order(default_order, model._key) \
.fetch_page_async(args.limit or _DEFAULT_LIMIT, start_cursor=args.cursor)
total_count_future = model.query(filters=filter_node).count_async(
keys_only=True)
items, next_cursor, more = items_future.get_result()
return items, next_cursor, more, total_count_future
def do(self, payload):
attempt = int(self.request.headers.get('X-AppEngine-TaskExecutionCount', 0))
seg_start = payload['seg_start']
# Check _MAX_BUILD_ID again in case the task was already scheduled.
seg_end = min(_MAX_BUILD_ID, payload['seg_end'])
start_from = payload.get('start_from', seg_start)
proc = payload['proc']
proc_def = _get_proc(proc['name'])
logging.info('range %d-%d', seg_start, seg_end)
logging.info('starting from %s', start_from)
if attempt > 0:
logging.warning('attempt %d', attempt)
q = ndb.Query(
kind=proc_def['entity_kind'],
filters=ndb.ConjunctionNode(
ndb.FilterNode('__key__', '>=', ndb.Key(model.Build, start_from)),
ndb.FilterNode('__key__', '<=', ndb.Key(model.Build, seg_end)),
),
)
iterator = q.iter(keys_only=proc_def['keys_only'])
entity_count = [0]
def iterate_segment():
# Datastore query timeout is 60 sec. Limit it to 50 sec.
deadline = utils.utcnow() + datetime.timedelta(seconds=50)
while (utils.utcnow() < deadline and
entity_count[0] < self.ENTITY_LIMIT and iterator.has_next()):
yield iterator.next()
entity_count[0] += 1
proc_def['func'](iterate_segment(), proc['payload'])
logging.info('processed %d entities', entity_count[0])
if iterator.has_next():
logging.info('enqueuing a task for the next iteration...')
build_key = (
iterator.next() if proc_def['keys_only'] else iterator.next().key
)
while build_key.parent() is not None:
build_key = build_key.parent()
p = payload.copy()
p['iteration'] += 1
p['start_from'] = build_key.id()
seg_len = seg_end - seg_start + 1
percent = 100 * (p['start_from'] - seg_start) / seg_len
try:
self._recurse([(
'{job_id}-{seg_index}-{iteration}',
'segment/seg:{seg_index}-percent:%d' % percent,
p,
)])
except (taskqueue.TaskAlreadyExistsError,
taskqueue.TombstonedTaskError): # pragma: no cover
pass
return
started_time = utils.timestamp_to_datetime(payload['started_ts'])
logging.info(
'segment %d is done in %s',
payload['seg_index'],
utils.utcnow() - started_time,
)
