def filter_ndb_query(self, query, filters=None): """Add query filter to restrict to this key range. Args: query: An ndb.Query instance. filters: optional list of filters to apply to the query. Each filter is a tuple: (<property_name_as_str>, <query_operation_as_str>, <value>). User filters are applied first. Returns: The input query restricted to this key range. """ assert _IsNdbQuery(query) if filters: for f in filters: query = query.filter(ndb.FilterNode(*f)) if self.include_start: start_comparator = ">=" else: start_comparator = ">" if self.include_end: end_comparator = "<=" else: end_comparator = "<" if self.key_start: query = query.filter( ndb.FilterNode("__key__", start_comparator, self.key_start)) if self.key_end: query = query.filter( ndb.FilterNode("__key__", end_comparator, self.key_end)) return query
def filter_ndb_query(self, query): """Add query filter to restrict to this key range. Args: query: An ndb.Query instance. Returns: The input query restricted to this key range. """ assert _IsNdbQuery(query) if self.include_start: start_comparator = ">=" else: start_comparator = ">" if self.include_end: end_comparator = "<=" else: end_comparator = "<" if self.key_start: query = query.filter( ndb.FilterNode("__key__", start_comparator, self.key_start)) if self.key_end: query = query.filter( ndb.FilterNode("__key__", end_comparator, self.key_end)) return query
def _QueryModel(self, search_dict, ancestor=None): """Queries the model class for field-value pairs. Args: search_dict: A dictionary mapping from field name to search by to the search term. ancestor: ndb.Key, If provided, the ancestor for the query. Returns: The model query. Raises: QueryError: If the queried field is not a property of the model. QueryTypeError: If search_term does not match the type of the search_base model property. """ filter_nodes = [] for search_base, search_term in search_dict.items(): field_name = upvote_utils.CamelToSnakeCase(search_base) # If the model class offers a translation function for property queries, # invoke it and set the field and search term to the result. try: field_name, search_term = self.MODEL_CLASS.TranslatePropertyQuery( field_name, search_term) except AttributeError: pass else: logging.info('Converted query to (%s = %s)', field_name, search_term) # Check for the property on the model itself (as opposed to, say, catching # a getattr exception) to ensure that the field being accessed is an ndb # property as opposed to a Python attribute. if not model_utils.HasProperty(self.MODEL_CLASS, field_name): raise QueryError('Invalid searchBase %s' % field_name) field = getattr(self.MODEL_CLASS, field_name) # If the field is of a non-string type, attempt to coerce the argument to # conform to this type search_term = _CoerceQueryParam(field, search_term) filter_nodes.append(ndb.FilterNode(field_name, '=', search_term)) query = self.MODEL_CLASS.query(ancestor=ancestor) if filter_nodes: query = query.filter(ndb.AND(*filter_nodes)) return query
def make_query(self, ns): """Make a query of entities within this range. Query options are not supported. They should be specified when the query is run. Args: ns: namespace of this query. Returns: a db.Query or ndb.Query, depends on the model class's type. """ if issubclass(self.model_class, db.Model): query = db.Query(self.model_class, namespace=ns) for f in self.filters: query.filter("%s %s" % (f[0], f[1]), f[2]) else: query = self.model_class.query(namespace=ns) for f in self.filters: query = query.filter(ndb.FilterNode(*f)) return query
def get_list_for_model(args, model, or_filters=None, and_filters=None): """ Creates a list for the model :param args: :param model: :param default_order: :param or_filters: :return: items, next_cursor, more, total_count_future """ default_order = -model.modified if args.get('order'): order_prop = model._properties.get(args.get('order')) if order_prop: default_order = -order_prop if args.get('order_reverse', False) else order_prop filter_node = None or_filters = or_filters or [] if not isinstance(or_filters, list): or_filters = [or_filters] if args.get('filter'): for filter in args.get('filter'): # split filter into its components: e.g. filter=course,filter=2017,timestamp>12241231 _and_filters = and_filters or [] for item in filter.split('+'): comparison = None result = [] for delimiter in [ COMP_EQUALITY, COMP_GREATER_THAN, COMP_LESS_THAN ]: result = [s.strip() for s in re.split(delimiter, item)] if len(result) == 2: comparison = delimiter break if comparison and result[0] in model._properties: property = model.get_all_properties().get(result[0]) value = result[1] if property and property.get('converter', None): value = property.get('converter')(result[1]) _and_filters.append( ndb.FilterNode(result[0], comparison, value)) if comparison != COMP_EQUALITY: # sort order must be altered args['order'] = getattr(model, result[0]) if len(_and_filters) > 0: or_filters.append(ndb.AND(*_and_filters)) elif and_filters: or_filters.append(ndb.AND(*and_filters)) if len(or_filters) == 1: filter_node = or_filters[0] elif len(or_filters) > 0: filter_node = ndb.OR(*or_filters) items_future = model.query(filters=filter_node).order(default_order, model._key) \ .fetch_page_async(args.limit or _DEFAULT_LIMIT, start_cursor=args.cursor) total_count_future = model.query(filters=filter_node).count_async( keys_only=True) items, next_cursor, more = items_future.get_result() return items, next_cursor, more, total_count_future
def do(self, payload): attempt = int(self.request.headers.get('X-AppEngine-TaskExecutionCount', 0)) seg_start = payload['seg_start'] # Check _MAX_BUILD_ID again in case the task was already scheduled. seg_end = min(_MAX_BUILD_ID, payload['seg_end']) start_from = payload.get('start_from', seg_start) proc = payload['proc'] proc_def = _get_proc(proc['name']) logging.info('range %d-%d', seg_start, seg_end) logging.info('starting from %s', start_from) if attempt > 0: logging.warning('attempt %d', attempt) q = ndb.Query( kind=proc_def['entity_kind'], filters=ndb.ConjunctionNode( ndb.FilterNode('__key__', '>=', ndb.Key(model.Build, start_from)), ndb.FilterNode('__key__', '<=', ndb.Key(model.Build, seg_end)), ), ) iterator = q.iter(keys_only=proc_def['keys_only']) entity_count = [0] def iterate_segment(): # Datastore query timeout is 60 sec. Limit it to 50 sec. deadline = utils.utcnow() + datetime.timedelta(seconds=50) while (utils.utcnow() < deadline and entity_count[0] < self.ENTITY_LIMIT and iterator.has_next()): yield iterator.next() entity_count[0] += 1 proc_def['func'](iterate_segment(), proc['payload']) logging.info('processed %d entities', entity_count[0]) if iterator.has_next(): logging.info('enqueuing a task for the next iteration...') build_key = ( iterator.next() if proc_def['keys_only'] else iterator.next().key ) while build_key.parent() is not None: build_key = build_key.parent() p = payload.copy() p['iteration'] += 1 p['start_from'] = build_key.id() seg_len = seg_end - seg_start + 1 percent = 100 * (p['start_from'] - seg_start) / seg_len try: self._recurse([( '{job_id}-{seg_index}-{iteration}', 'segment/seg:{seg_index}-percent:%d' % percent, p, )]) except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError): # pragma: no cover pass return started_time = utils.timestamp_to_datetime(payload['started_ts']) logging.info( 'segment %d is done in %s', payload['seg_index'], utils.utcnow() - started_time, )