def SetupIndexes(app_id, root_path): """Ensure that the set of existing composite indexes matches index.yaml. Note: this is similar to the algorithm used by the admin console for the same purpose. Args: app_id: Application ID being served. root_path: Path to the root of the application. """ index_yaml_file = os.path.join(root_path, 'index.yaml') try: fh = open(index_yaml_file, 'r') except IOError: index_yaml_data = None else: try: index_yaml_data = fh.read() finally: fh.close() indexes = [] if index_yaml_data is not None: index_defs = datastore_index.ParseIndexDefinitions(index_yaml_data) if index_defs is not None: indexes = index_defs.indexes if indexes is None: indexes = [] requested_indexes = datastore_index.IndexDefinitionsToProtos(app_id, indexes) existing_indexes = datastore_admin.GetIndices(app_id) requested = dict((x.definition().Encode(), x) for x in requested_indexes) existing = dict((x.definition().Encode(), x) for x in existing_indexes) created = 0 for key, index in requested.iteritems(): if key not in existing: datastore_admin.CreateIndex(index) created += 1 deleted = 0 for key, index in existing.iteritems(): if key not in requested: datastore_admin.DeleteIndex(index) deleted += 1 if created or deleted: logging.info("Created %d and deleted %d index(es); total %d", created, deleted, len(requested))
def main(): print 'Checking indexes...' configure_remote_api() interval = 10 # seconds. building = True while building: # Start with a fresh run: maybe we're done building? building = False for index in datastore_admin.GetIndices('s~edufocal-app'): # If any single index is building, we're not done. # Sleep for a bit and then this cycle should repeat. if not index.has_state() or index.state() != index.READ_WRITE: building = True print 'Indexes are still building... Waiting %s seconds.' % interval time.sleep(interval) break print 'All indexes are up to date.'
def __AggregateCompositeIndices(self, proto, namespace, kind_name, entity_key_size): """Aggregate statistics of composite indexes for an entity.""" composite_indices = datastore_admin.GetIndices(self.app_id) for index in composite_indices: definition = index.definition() if kind_name != definition.entity_type(): continue index_size, index_count = self.__GetCompositeIndexStat( definition, proto, namespace, kind_name, entity_key_size) if index_count == 0: continue name_id = namespace if not name_id: name_id = 1 self.__Increment(self.whole_app_stats, 0, _GLOBAL_KEY, 0, composite_index_count=index_count, composite_index_size=index_size) self.__Increment(self.whole_app_stats, 0, (stats.NamespaceStat, name_id, ''), 0, composite_index_count=index_count, composite_index_size=index_size, subject_namespace=namespace) self.__Increment( self.namespace_stats, 0, (stats.NamespaceGlobalStat, 'total_entity_usage', namespace), 0, composite_index_count=index_count, composite_index_size=index_size) self.__Increment(self.whole_app_stats, 0, (stats.KindStat, kind_name, ''), 0, composite_index_count=index_count, composite_index_size=index_size, kind_name=kind_name) self.__Increment(self.namespace_stats, 0, (stats.NamespaceKindStat, kind_name, namespace), 0, composite_index_count=index_count, composite_index_size=index_size, kind_name=kind_name) index_id = index.id() self.__Increment(self.whole_app_stats, index_count, (stats.KindCompositeIndexStat, kind_name + '_%s' % index_id, ''), index_size, kind_name=kind_name, index_id=index_id) self.__Increment(self.namespace_stats, index_count, (stats.NamespaceKindCompositeIndexStat, kind_name + '_%s' % index_id, namespace), index_size, kind_name=kind_name, index_id=index_id)
def SetupIndexes(app_id, root_path): """Ensure that the set of existing composite indexes matches index.yaml. Note: this is similar to the algorithm used by the admin console for the same purpose. Args: app_id: Application ID being served. root_path: Path to the root of the application. """ index_yaml_file = os.path.join(root_path, 'index.yaml') global _cached_yaml if _cached_yaml[0] == index_yaml_file and os.path.exists( index_yaml_file) and os.path.getmtime( index_yaml_file) == _cached_yaml[1]: requested_indexes = _cached_yaml[2] else: try: index_yaml_mtime = os.path.getmtime(index_yaml_file) fh = open(index_yaml_file, 'r') except (OSError, IOError): index_yaml_data = None else: try: index_yaml_data = fh.read() finally: fh.close() requested_indexes = [] if index_yaml_data is not None: index_defs = datastore_index.ParseIndexDefinitions(index_yaml_data) if index_defs is not None and index_defs.indexes is not None: requested_indexes = datastore_index.IndexDefinitionsToProtos( app_id, index_defs.indexes) _cached_yaml = (index_yaml_file, index_yaml_mtime, requested_indexes) existing_indexes = datastore_admin.GetIndices(app_id) requested = dict((x.definition().Encode(), x) for x in requested_indexes) existing = dict((x.definition().Encode(), x) for x in existing_indexes) created = 0 for key, index in requested.iteritems(): if key not in existing: new_index = entity_pb.CompositeIndex() new_index.CopyFrom(index) id = datastore_admin.CreateIndex(new_index) new_index.set_id(id) new_index.set_state(entity_pb.CompositeIndex.READ_WRITE) datastore_admin.UpdateIndex(new_index) created += 1 deleted = 0 for key, index in existing.iteritems(): if key not in requested: datastore_admin.DeleteIndex(index) deleted += 1 if created or deleted: logging.info("Created %d and deleted %d index(es); total %d", created, deleted, len(requested))