def set_leaf(self, column, operator, value, is_pk_field, negated, lookup_name, namespace, target_field=None): assert column assert operator assert isinstance(is_pk_field, bool) assert isinstance(negated, bool) if operator == "iexact" and isinstance(target_field, AutoField): # When new instance is created, automatic primary key 'id' does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, # we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. operator = "exact" value = int(value) if is_pk_field: # If this is a primary key, we need to make sure that the value # we pass to the query is a datastore Key. We have to deal with IN queries here # because they aren't flattened until the DNF stage model = get_top_concrete_parent(target_field.model) table = model._meta.db_table if isinstance(value, (list, tuple)): value = [ datastore.Key.from_path(table, x, namespace=namespace) for x in value if x ] else: if (operator == "isnull" and value is True) or not value: # id=None will never return anything and # Empty strings and 0 are forbidden as keys self.will_never_return_results = True else: value = datastore.Key.from_path(table, value, namespace=namespace) column = "__key__" # Do any special index conversions necessary to perform this lookup special_indexer = get_indexer(target_field, operator) if special_indexer: if is_pk_field: column = model._meta.pk.column value = unicode(value.id_or_name()) add_special_index(target_field.model, column, special_indexer, operator, value) index_type = special_indexer.prepare_index_type(operator, value) value = special_indexer.prep_value_for_query(value) column = special_indexer.indexed_column_name(column, value, index_type) operator = special_indexer.prep_query_operator(operator) self.column = column self.operator = convert_operator(operator) self.value = value self.lookup_name = lookup_name
def django_instance_to_entity(connection, model, fields, raw, instance, check_null=True): from djangae.db.backends.appengine.indexing import special_indexes_for_column, get_indexer from djangae.db.backends.appengine import POLYMODEL_CLASS_ATTRIBUTE # uses_inheritance = False inheritance_root = get_top_concrete_parent(model) db_table = get_datastore_kind(inheritance_root) def value_from_instance(_instance, _field): value = get_prepared_db_value(connection, _instance, _field, raw) # If value is None, but there is a default, and the field is not nullable then we should populate it # Otherwise thing get hairy when you add new fields to models if value is None and _field.has_default() and not _field.null: value = connection.ops.value_for_db(_field.get_default(), _field) if check_null and (not _field.null and not _field.primary_key) and value is None: raise IntegrityError("You can't set %s (a non-nullable " "field) to None!" % _field.name) is_primary_key = False if _field.primary_key and _field.model == inheritance_root: is_primary_key = True return value, is_primary_key field_values = {} primary_key = None for field in fields: value, is_primary_key = value_from_instance(instance, field) if is_primary_key: primary_key = value else: field_values[field.column] = value # Add special indexed fields for index in special_indexes_for_column(model, field.column): indexer = get_indexer(field, index) values = indexer.prep_value_for_database(value, index) if values is None: continue if not hasattr(values, "__iter__"): values = [ values ] for v in values: column = indexer.indexed_column_name(field.column, v, index) if column in field_values: if not isinstance(field_values[column], list): field_values[column] = [ field_values[column], v ] else: field_values[column].append(v) else: field_values[column] = v kwargs = {} if primary_key: if isinstance(primary_key, (int, long)): kwargs["id"] = primary_key elif isinstance(primary_key, basestring): if len(primary_key) > 500: warnings.warn("Truncating primary key that is over 500 characters. " "THIS IS AN ERROR IN YOUR PROGRAM.", RuntimeWarning) primary_key = primary_key[:500] kwargs["name"] = primary_key else: raise ValueError("Invalid primary key value") namespace = connection.settings_dict.get("NAMESPACE") entity = datastore.Entity(db_table, namespace=namespace, **kwargs) entity.update(field_values) classes = get_concrete_db_tables(model) if len(classes) > 1: entity[POLYMODEL_CLASS_ATTRIBUTE] = list(set(classes)) return entity
def django_instance_to_entities(connection, fields, raw, instance, check_null=True, model=None): """ Converts a Django Model instance to an App Engine `Entity` Arguments: connection: Djangae appengine connection object fields: A list of fields to populate in the Entity raw: raw flag to pass to get_prepared_db_value instance: The Django model instance to convert check_null: Whether or not we should enforce NULL during conversion (throws an error if None is set on a non-nullable field) model: Model class to use instead of the instance one Returns: entity, [entity, entity, ...] Where the first result in the tuple is the primary entity, and the remaining entities are optionally descendents of the primary entity. This is useful for special indexes (e.g. contains) """ from djangae.db.backends.appengine.indexing import special_indexes_for_column, get_indexer, IgnoreForIndexing from djangae.db.backends.appengine import POLYMODEL_CLASS_ATTRIBUTE model = model or type(instance) inheritance_root = get_top_concrete_parent(model) db_table = get_datastore_kind(inheritance_root) def value_from_instance(_instance, _field): value = get_prepared_db_value(connection, _instance, _field, raw) # If value is None, but there is a default, and the field is not nullable then we should populate it # Otherwise thing get hairy when you add new fields to models if value is None and _field.has_default() and not _field.null: value = connection.ops.value_for_db(_field.get_default(), _field) if check_null and (not _field.null and not _field.primary_key) and value is None: raise IntegrityError("You can't set %s (a non-nullable " "field) to None!" % _field.name) is_primary_key = False if _field.primary_key and _field.model == inheritance_root: is_primary_key = True return value, is_primary_key field_values = {} primary_key = None descendents = [] fields_to_unindex = set() for field in fields: value, is_primary_key = value_from_instance(instance, field) if is_primary_key: primary_key = value else: field_values[field.column] = value # Add special indexed fields for index in special_indexes_for_column(model, field.column): indexer = get_indexer(field, index) unindex = False try: values = indexer.prep_value_for_database(value, index, model=model, column=field.column) except IgnoreForIndexing as e: # We mark this value as being wiped out for indexing unindex = True values = e.processed_value if not hasattr(values, "__iter__"): values = [values] # If the indexer returns additional entities (instead of indexing a special column) # then just store those entities if indexer.PREP_VALUE_RETURNS_ENTITIES: descendents.extend(values) else: for i, v in enumerate(values): column = indexer.indexed_column_name( field.column, v, index) if unindex: fields_to_unindex.add(column) continue # If the column already exists in the values, then we convert it to a # list and append the new value if column in field_values: if not isinstance(field_values[column], list): field_values[column] = [field_values[column], v] else: field_values[column].append(v) else: # Otherwise we just set the column to the value field_values[column] = v kwargs = {} if primary_key: if isinstance(primary_key, six.integer_types): kwargs["id"] = primary_key elif isinstance(primary_key, six.string_types): kwargs["name"] = primary_key else: raise ValueError("Invalid primary key value") namespace = connection.settings_dict.get("NAMESPACE") entity = datastore.Entity(db_table, namespace=namespace, **kwargs) entity.update(field_values) if fields_to_unindex: entity._properties_to_remove = fields_to_unindex classes = get_concrete_db_tables(model) if len(classes) > 1: entity[POLYMODEL_CLASS_ATTRIBUTE] = list(set(classes)) return entity, descendents
def django_instance_to_entities(connection, fields, raw, instance, check_null=True, model=None): """ Converts a Django Model instance to an App Engine `Entity` Arguments: connection: Djangae appengine connection object fields: A list of fields to populate in the Entity raw: raw flag to pass to get_prepared_db_value instance: The Django model instance to convert check_null: Whether or not we should enforce NULL during conversion (throws an error if None is set on a non-nullable field) model: Model class to use instead of the instance one Returns: entity, [entity, entity, ...] Where the first result in the tuple is the primary entity, and the remaining entities are optionally descendents of the primary entity. This is useful for special indexes (e.g. contains) """ from djangae.db.backends.appengine.indexing import special_indexes_for_column, get_indexer, IgnoreForIndexing from djangae.db.backends.appengine import POLYMODEL_CLASS_ATTRIBUTE model = model or type(instance) inheritance_root = get_top_concrete_parent(model) db_table = get_datastore_kind(inheritance_root) def value_from_instance(_instance, _field): value = get_prepared_db_value(connection, _instance, _field, raw) # If value is None, but there is a default, and the field is not nullable then we should populate it # Otherwise thing get hairy when you add new fields to models if value is None and _field.has_default() and not _field.null: value = connection.ops.value_for_db(_field.get_default(), _field) if check_null and (not _field.null and not _field.primary_key) and value is None: raise IntegrityError("You can't set %s (a non-nullable " "field) to None!" % _field.name) is_primary_key = False if _field.primary_key and _field.model == inheritance_root: is_primary_key = True return value, is_primary_key field_values = {} primary_key = None descendents = [] fields_to_unindex = set() for field in fields: value, is_primary_key = value_from_instance(instance, field) if is_primary_key: primary_key = value else: field_values[field.column] = value # Add special indexed fields for index in special_indexes_for_column(model, field.column): indexer = get_indexer(field, index) unindex = False try: values = indexer.prep_value_for_database(value, index, model=model, column=field.column) except IgnoreForIndexing as e: # We mark this value as being wiped out for indexing unindex = True values = e.processed_value if not hasattr(values, "__iter__"): values = [values] # If the indexer returns additional entities (instead of indexing a special column) # then just store those entities if indexer.PREP_VALUE_RETURNS_ENTITIES: descendents.extend(values) else: for i, v in enumerate(values): column = indexer.indexed_column_name(field.column, v, index) if unindex: fields_to_unindex.add(column) continue # If the column already exists in the values, then we convert it to a # list and append the new value if column in field_values: if not isinstance(field_values[column], list): field_values[column] = [field_values[column], v] else: field_values[column].append(v) else: # Otherwise we just set the column to the value field_values[column] = v kwargs = {} if primary_key: if isinstance(primary_key, six.integer_types): kwargs["id"] = primary_key elif isinstance(primary_key, six.string_types): kwargs["name"] = primary_key else: raise ValueError("Invalid primary key value") namespace = connection.settings_dict.get("NAMESPACE") entity = datastore.Entity(db_table, namespace=namespace, **kwargs) entity.update(field_values) if fields_to_unindex: entity._properties_to_remove = fields_to_unindex classes = get_concrete_db_tables(model) if len(classes) > 1: entity[POLYMODEL_CLASS_ATTRIBUTE] = list(set(classes)) return entity, descendents
def django_instance_to_entity(connection, model, fields, raw, instance, check_null=True): from djangae.db.backends.appengine.indexing import special_indexes_for_column, get_indexer from djangae.db.backends.appengine import POLYMODEL_CLASS_ATTRIBUTE # uses_inheritance = False inheritance_root = get_top_concrete_parent(model) db_table = get_datastore_kind(inheritance_root) def value_from_instance(_instance, _field): value = get_prepared_db_value(connection, _instance, _field, raw) # If value is None, but there is a default, and the field is not nullable then we should populate it # Otherwise thing get hairy when you add new fields to models if value is None and _field.has_default() and not _field.null: value = connection.ops.value_for_db(_field.get_default(), _field) if check_null and (not _field.null and not _field.primary_key) and value is None: raise IntegrityError("You can't set %s (a non-nullable " "field) to None!" % _field.name) is_primary_key = False if _field.primary_key and _field.model == inheritance_root: is_primary_key = True return value, is_primary_key field_values = {} primary_key = None for field in fields: value, is_primary_key = value_from_instance(instance, field) if is_primary_key: primary_key = value else: field_values[field.column] = value # Add special indexed fields for index in special_indexes_for_column(model, field.column): indexer = get_indexer(field, index) values = indexer.prep_value_for_database(value, index) if values is None: continue if not hasattr(values, "__iter__"): values = [values] for v in values: column = indexer.indexed_column_name(field.column, v, index) if column in field_values: if not isinstance(field_values[column], list): field_values[column] = [field_values[column], v] else: field_values[column].append(v) else: field_values[column] = v kwargs = {} if primary_key: if isinstance(primary_key, (int, long)): kwargs["id"] = primary_key elif isinstance(primary_key, basestring): if len(primary_key) > 500: warnings.warn( "Truncating primary key that is over 500 characters. " "THIS IS AN ERROR IN YOUR PROGRAM.", RuntimeWarning, ) primary_key = primary_key[:500] kwargs["name"] = primary_key else: raise ValueError("Invalid primary key value") namespace = connection.settings_dict.get("NAMESPACE") entity = datastore.Entity(db_table, namespace=namespace, **kwargs) entity.update(field_values) classes = get_concrete_db_tables(model) if len(classes) > 1: entity[POLYMODEL_CLASS_ATTRIBUTE] = list(set(classes)) return entity
def set_leaf(self, column, operator, value, is_pk_field, negated, lookup_name, namespace, target_field=None): assert column assert operator assert isinstance(is_pk_field, bool) assert isinstance(negated, bool) if operator == "iexact" and isinstance(target_field, AutoField): # When new instance is created, automatic primary key 'id' # does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, # we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. operator = "exact" value = int(value) if is_pk_field: # If this is a primary key, we need to make sure that the value # we pass to the query is a datastore Key. We have to deal with IN queries here # because they aren't flattened until the DNF stage model = get_top_concrete_parent(target_field.model) table = model._meta.db_table if isinstance(value, (list, tuple)): value = [ rpc.Key.from_path(table, x, namespace=namespace) for x in value if x ] else: # Django 1.11 has operators as symbols, earlier versions use "exact" etc. if (operator == "isnull" and value is True) or ( operator in ("exact", "lt", "lte", "<", "<=", "=") and not value): # id=None will never return anything and # Empty strings and 0 are forbidden as keys self.will_never_return_results = True elif operator in ("gt", "gte", ">", ">=") and not value: # If the value is 0 or "", then we need to manipulate the value and operator here to # get the right result (given that both are invalid keys) so for both we return # >= 1 or >= "\0" for strings if isinstance(value, six.integer_types): value = 1 else: value = "\0" value = rpc.Key.from_path(table, value, namespace=namespace) operator = "gte" else: value = rpc.Key.from_path(table, value, namespace=namespace) column = "__key__" # Do any special index conversions necessary to perform this lookup special_indexer = get_indexer(target_field, operator) if special_indexer: if is_pk_field: column = model._meta.pk.column value = unicode(value.id_or_name()) add_special_index(target_field.model, column, special_indexer, operator, value) index_type = special_indexer.prepare_index_type(operator, value) value = special_indexer.prep_value_for_query( value, model=target_field.model, column=column, connection=connections[self.using]) column = special_indexer.indexed_column_name( column, value, index_type) operator = special_indexer.prep_query_operator(operator) self.column = column self.operator = convert_operator(operator) self.value = value self.lookup_name = lookup_name
def set_leaf( self, column, operator, value, is_pk_field, negated, lookup_name, namespace, target_field=None): assert column assert operator assert isinstance(is_pk_field, bool) assert isinstance(negated, bool) if operator == "iexact" and isinstance(target_field, AutoField): # When new instance is created, automatic primary key 'id' # does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, # we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. operator = "exact" value = int(value) if is_pk_field: # If this is a primary key, we need to make sure that the value # we pass to the query is a datastore Key. We have to deal with IN queries here # because they aren't flattened until the DNF stage model = get_top_concrete_parent(target_field.model) table = model._meta.db_table if isinstance(value, (list, tuple)): value = [ rpc.Key.from_path(table, x, namespace=namespace) for x in value if x ] else: # Django 1.11 has operators as symbols, earlier versions use "exact" etc. if (operator == "isnull" and value is True) or (operator in ("exact", "lt", "lte", "<", "<=", "=") and not value): # id=None will never return anything and # Empty strings and 0 are forbidden as keys self.will_never_return_results = True elif operator in ("gt", "gte", ">", ">=") and not value: # If the value is 0 or "", then we need to manipulate the value and operator here to # get the right result (given that both are invalid keys) so for both we return # >= 1 or >= "\0" for strings if isinstance(value, six.integer_types): value = 1 else: value = "\0" value = rpc.Key.from_path(table, value, namespace=namespace) operator = "gte" else: value = rpc.Key.from_path(table, value, namespace=namespace) column = "__key__" # Do any special index conversions necessary to perform this lookup special_indexer = get_indexer(target_field, operator) if special_indexer: if is_pk_field: column = model._meta.pk.column value = unicode(value.id_or_name()) add_special_index(target_field.model, column, special_indexer, operator, value) index_type = special_indexer.prepare_index_type(operator, value) value = special_indexer.prep_value_for_query( value, model=target_field.model, column=column, connection=connections[self.using] ) column = special_indexer.indexed_column_name(column, value, index_type) operator = special_indexer.prep_query_operator(operator) self.column = column self.operator = convert_operator(operator) self.value = value self.lookup_name = lookup_name