def SetupIndexes(app_id, root_path):
"""Ensure that the set of existing composite indexes matches index.yaml.
Note: this is similar to the algorithm used by the admin console for
the same purpose.
Args:
app_id: Application ID being served.
root_path: Path to the root of the application.
"""
index_yaml_file = os.path.join(root_path, 'index.yaml')
try:
fh = open(index_yaml_file, 'r')
except IOError:
index_yaml_data = None
else:
try:
index_yaml_data = fh.read()
finally:
fh.close()
indexes = []
if index_yaml_data is not None:
index_defs = datastore_index.ParseIndexDefinitions(index_yaml_data)
if index_defs is not None:
indexes = index_defs.indexes
if indexes is None:
indexes = []
requested_indexes = datastore_index.IndexDefinitionsToProtos(app_id, indexes)
existing_indexes = datastore_admin.GetIndices(app_id)
requested = dict((x.definition().Encode(), x) for x in requested_indexes)
existing = dict((x.definition().Encode(), x) for x in existing_indexes)
created = 0
for key, index in requested.iteritems():
if key not in existing:
datastore_admin.CreateIndex(index)
created += 1
deleted = 0
for key, index in existing.iteritems():
if key not in requested:
datastore_admin.DeleteIndex(index)
deleted += 1
if created or deleted:
logging.info("Created %d and deleted %d index(es); total %d",
created, deleted, len(requested))
def main():
print 'Checking indexes...'
configure_remote_api()
interval = 10 # seconds.
building = True
while building:
# Start with a fresh run: maybe we're done building?
building = False
for index in datastore_admin.GetIndices('s~edufocal-app'):
# If any single index is building, we're not done.
# Sleep for a bit and then this cycle should repeat.
if not index.has_state() or index.state() != index.READ_WRITE:
building = True
print 'Indexes are still building... Waiting %s seconds.' % interval
time.sleep(interval)
break
print 'All indexes are up to date.'
def __AggregateCompositeIndices(self, proto, namespace, kind_name,
entity_key_size):
"""Aggregate statistics of composite indexes for an entity."""
composite_indices = datastore_admin.GetIndices(self.app_id)
for index in composite_indices:
definition = index.definition()
if kind_name != definition.entity_type():
continue
index_size, index_count = self.__GetCompositeIndexStat(
definition, proto, namespace, kind_name, entity_key_size)
if index_count == 0:
continue
name_id = namespace
if not name_id:
name_id = 1
self.__Increment(self.whole_app_stats,
0,
_GLOBAL_KEY,
0,
composite_index_count=index_count,
composite_index_size=index_size)
self.__Increment(self.whole_app_stats,
0, (stats.NamespaceStat, name_id, ''),
0,
composite_index_count=index_count,
composite_index_size=index_size,
subject_namespace=namespace)
self.__Increment(
self.namespace_stats,
0,
(stats.NamespaceGlobalStat, 'total_entity_usage', namespace),
0,
composite_index_count=index_count,
composite_index_size=index_size)
self.__Increment(self.whole_app_stats,
0, (stats.KindStat, kind_name, ''),
0,
composite_index_count=index_count,
composite_index_size=index_size,
kind_name=kind_name)
self.__Increment(self.namespace_stats,
0,
(stats.NamespaceKindStat, kind_name, namespace),
0,
composite_index_count=index_count,
composite_index_size=index_size,
kind_name=kind_name)
index_id = index.id()
self.__Increment(self.whole_app_stats,
index_count, (stats.KindCompositeIndexStat,
kind_name + '_%s' % index_id, ''),
index_size,
kind_name=kind_name,
index_id=index_id)
self.__Increment(self.namespace_stats,
index_count,
(stats.NamespaceKindCompositeIndexStat,
kind_name + '_%s' % index_id, namespace),
index_size,
kind_name=kind_name,
index_id=index_id)
def SetupIndexes(app_id, root_path):
"""Ensure that the set of existing composite indexes matches index.yaml.
Note: this is similar to the algorithm used by the admin console for
the same purpose.
Args:
app_id: Application ID being served.
root_path: Path to the root of the application.
"""
index_yaml_file = os.path.join(root_path, 'index.yaml')
global _cached_yaml
if _cached_yaml[0] == index_yaml_file and os.path.exists(
index_yaml_file) and os.path.getmtime(
index_yaml_file) == _cached_yaml[1]:
requested_indexes = _cached_yaml[2]
else:
try:
index_yaml_mtime = os.path.getmtime(index_yaml_file)
fh = open(index_yaml_file, 'r')
except (OSError, IOError):
index_yaml_data = None
else:
try:
index_yaml_data = fh.read()
finally:
fh.close()
requested_indexes = []
if index_yaml_data is not None:
index_defs = datastore_index.ParseIndexDefinitions(index_yaml_data)
if index_defs is not None and index_defs.indexes is not None:
requested_indexes = datastore_index.IndexDefinitionsToProtos(
app_id, index_defs.indexes)
_cached_yaml = (index_yaml_file, index_yaml_mtime,
requested_indexes)
existing_indexes = datastore_admin.GetIndices(app_id)
requested = dict((x.definition().Encode(), x) for x in requested_indexes)
existing = dict((x.definition().Encode(), x) for x in existing_indexes)
created = 0
for key, index in requested.iteritems():
if key not in existing:
new_index = entity_pb.CompositeIndex()
new_index.CopyFrom(index)
id = datastore_admin.CreateIndex(new_index)
new_index.set_id(id)
new_index.set_state(entity_pb.CompositeIndex.READ_WRITE)
datastore_admin.UpdateIndex(new_index)
created += 1
deleted = 0
for key, index in existing.iteritems():
if key not in requested:
datastore_admin.DeleteIndex(index)
deleted += 1
if created or deleted:
logging.info("Created %d and deleted %d index(es); total %d", created,
deleted, len(requested))
