def build_entity_query(self): """ Builds a :class:`sqla:sqlalchemy.orm.query.Query` object for this entity (an instance of :class:`sir.schema.searchentities.SearchEntity`) that eagerly loads the values of all search fields. :rtype: :class:`sqla:sqlalchemy.orm.query.Query` """ root_model = self.model query = Query(root_model) paths = [field.paths for field in self.fields] if (config.CFG.getboolean("sir", "wscompat") and self.extrapaths is not None): paths.extend([self.extrapaths]) merged_paths = merge_paths(paths) for field_paths in paths: for path in field_paths: current_merged_path = merged_paths model = root_model load = Load(model) split_path = path.split(".") for pathelem in split_path: current_merged_path = current_merged_path[pathelem] column = getattr(model, pathelem) prop = column.property if isinstance(prop, RelationshipProperty): pk = column.mapper.primary_key[0].name if prop.direction == ONETOMANY: load = load.subqueryload(pathelem) elif prop.direction == MANYTOONE: load = load.joinedload(pathelem) else: load = load.defaultload(pathelem) required_columns = current_merged_path.keys() required_columns.append(pk) # Get the mapper class of the current element of the path so # the next iteration can access it. model = prop.mapper.class_ logger.debug("Loading only %s on %s", required_columns, model) load = defer_everything_but(class_mapper(model), load, *required_columns) query = query.options(load) return query
def iter_data_entity(db, entity, condition=None, batches=True): query = db.query(entity.model) if condition is not None: query = query.filter(condition) for field in entity.fields: load = Load(entity.model) model = field.entity.model for attr in field.key.split('.'): prop = getattr(model, attr).property if isinstance(prop, RelationshipProperty): if prop.direction == ONETOMANY: load = load.subqueryload(attr) elif prop.direction == MANYTOONE: load = load.joinedload(attr) else: load = load.defaultload(attr) model = prop.mapper.class_ query = query.options(load) queries = [] if batches: min_id = query.with_entities(sql.func.min(entity.model.id)).scalar() max_id = query.with_entities(sql.func.max(entity.model.id)).scalar() for i in range(min_id, max_id + 1, BATCH_SIZE): queries.append( query.filter(entity.model.id.between(i, i + BATCH_SIZE - 1))) else: queries.append(query) for query in queries: for item in query.order_by(entity.model.id): data = set([ ('id', '{0}:{1}'.format(entity.name, item.id)), ('kind', entity.name), ]) for field in entity.fields: for value in get_field_value(item, field.key): if value is not None and value != '': data.add((field.name, value)) yield item.id, data
def build_entity_query(self): """ Builds a :class:`sqla:sqlalchemy.orm.query.Query` object for this entity (an instance of :class:`sir.schema.searchentities.SearchEntity`) that eagerly loads the values of all search fields. :rtype: :class:`sqla:sqlalchemy.orm.query.Query` """ root_model = self.model query = Query(root_model) paths = [field.paths for field in self.fields] if (config.CFG.getboolean("sir", "wscompat") and self.extrapaths is not None): paths.extend([self.extrapaths]) merged_paths = merge_paths(paths) for field_paths in paths: for path in field_paths: current_merged_path = merged_paths model = root_model load = Load(model) split_path = path.split(".") for pathelem in split_path: current_merged_path = current_merged_path[pathelem] column = getattr(model, pathelem) # __tablename__s in annotation paths if (not isinstance(column, InstrumentedAttribute) and not isinstance(column, CompositeProperty)): break prop = column.property if isinstance(prop, RelationshipProperty): pk = column.mapper.primary_key[0].name if prop.direction == ONETOMANY: load = load.subqueryload(pathelem) elif prop.direction == MANYTOONE: load = load.joinedload(pathelem) else: load = load.defaultload(pathelem) required_columns = current_merged_path.keys() required_columns.append(pk) # Get the mapper class of the current element of the # path so the next iteration can access it. model = prop.mapper.class_ # For composite properties, load the columns they # consist of because eagerly loading a composite # property doesn't load automatically load them. composite_columns = filter( partial(is_composite_column, model), required_columns) for composite_column in composite_columns: composite_parts = (c.name for c in getattr(model, composite_column). property.columns) logger.debug("Loading %s instead of %s on %s", composite_parts, composite_column, model) required_columns.remove(composite_column) required_columns.extend(composite_parts) logger.debug("Loading only %s on %s", required_columns, model) load = defer_everything_but(class_mapper(model), load, *required_columns) query = query.options(load) if self.extraquery is not None: query = self.extraquery(query) return query
def build_entity_query(self): """ Builds a :class:`sqla:sqlalchemy.orm.query.Query` object for this entity (an instance of :class:`sir.schema.searchentities.SearchEntity`) that eagerly loads the values of all search fields. :rtype: :class:`sqla:sqlalchemy.orm.query.Query` """ root_model = self.model query = Query(root_model) paths = [field.paths for field in self.fields] if (config.CFG.getboolean("sir", "wscompat") and self.extrapaths is not None): paths.extend([self.extrapaths]) merged_paths = merge_paths(paths) for field_paths in paths: for path in field_paths: current_merged_path = merged_paths model = root_model load = Load(model) split_path = path.split(".") for pathelem in split_path: current_merged_path = current_merged_path[pathelem] column = getattr(model, pathelem) # __tablename__s in annotation paths if (not isinstance(column, InstrumentedAttribute) and not isinstance(column, CompositeProperty)): break prop = column.property if isinstance(prop, RelationshipProperty): pk = column.mapper.primary_key[0].name if prop.direction == ONETOMANY: load = load.subqueryload(pathelem) elif prop.direction == MANYTOONE: load = load.joinedload(pathelem) else: load = load.defaultload(pathelem) required_columns = current_merged_path.keys() required_columns.append(pk) # Get the mapper class of the current element of the # path so the next iteration can access it. model = prop.mapper.class_ # For composite properties, load the columns they # consist of because eagerly loading a composite # property doesn't load automatically load them. composite_columns = filter( partial(is_composite_column, model), required_columns) for composite_column in composite_columns: composite_parts = (c.name for c in getattr( model, composite_column).property.columns) logger.debug("Loading %s instead of %s on %s", composite_parts, composite_column, model) required_columns.remove(composite_column) required_columns.extend(composite_parts) logger.debug("Loading only %s on %s", required_columns, model) load = defer_everything_but(class_mapper(model), load, *required_columns) query = query.options(load) if self.extraquery is not None: query = self.extraquery(query) return query