def UpdateIndexYaml(self, openfile=open):
"""Update index.yaml.
Args:
openfile: Used for dependency injection.
We only ever write to index.yaml if either:
- it doesn't exist yet; or
- it contains an 'AUTOGENERATED' comment.
All indexes *before* the AUTOGENERATED comment will be written
back unchanged. All indexes *after* the AUTOGENERATED comment
will be updated with the latest query counts (query counts are
reset by --clear_datastore). Indexes that aren't yet in the file
will be appended to the AUTOGENERATED section.
We keep track of some data in order to avoid doing repetitive work:
- if index.yaml is fully manual, we keep track of its mtime to
avoid parsing it over and over;
- we keep track of the number of keys in the history dict since
the last time we updated index.yaml (or decided there was
nothing to update).
"""
index_yaml_file = os.path.join(self.root_path, 'index.yaml')
try:
index_yaml_mtime = os.path.getmtime(index_yaml_file)
except os.error:
index_yaml_mtime = None
index_yaml_changed = (index_yaml_mtime != self.index_yaml_mtime)
self.index_yaml_mtime = index_yaml_mtime
datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
query_history = datastore_stub.QueryHistory()
history_changed = (len(query_history) != self.last_history_size)
self.last_history_size = len(query_history)
if not (index_yaml_changed or history_changed):
logging.debug('No need to update index.yaml')
return
if self.index_yaml_is_manual and not index_yaml_changed:
logging.debug('Will not update manual index.yaml')
return
if index_yaml_mtime is None:
index_yaml_data = None
else:
try:
fh = open(index_yaml_file, 'r')
except IOError:
index_yaml_data = None
else:
try:
index_yaml_data = fh.read()
finally:
fh.close()
self.index_yaml_is_manual = (index_yaml_data is not None
and AUTO_MARKER not in index_yaml_data)
if self.index_yaml_is_manual:
logging.info('Detected manual index.yaml, will not update')
return
if index_yaml_data is None:
all_indexes = None
else:
try:
all_indexes = datastore_index.ParseIndexDefinitions(
index_yaml_data)
except yaml_errors.EventListenerError, e:
logging.error('Error parsing %s:\n%s', index_yaml_file, e)
return
except Exception, err:
logging.error('Error parsing %s:\n%s.%s: %s', index_yaml_file,
err.__class__.__module__, err.__class__.__name__,
err)
return
except yaml_errors.EventListenerError, e:
logging.error('Error parsing %s:\n%s', index_yaml_file, e)
return
except Exception, err:
logging.error('Error parsing %s:\n%s.%s: %s', index_yaml_file,
err.__class__.__module__, err.__class__.__name__,
err)
return
if index_yaml_data is None:
manual_part, automatic_part = 'indexes:\n', ''
manual_indexes = None
else:
manual_part, automatic_part = index_yaml_data.split(AUTO_MARKER, 1)
try:
manual_indexes = datastore_index.ParseIndexDefinitions(
manual_part)
except Exception, err:
logging.error('Error parsing manual part of %s: %s',
index_yaml_file, err)
return
automatic_part = GenerateIndexFromHistory(query_history, all_indexes,
manual_indexes)
try:
fh = openfile(index_yaml_file, 'w')
except IOError, err:
logging.error('Can\'t write index.yaml: %s', err)
return
try:
def UpdateIndexYaml(self, openfile=open):
"""Update index.yaml.
Args:
openfile: Used for dependency injection.
We only ever write to index.yaml if either:
- it doesn't exist yet; or
- it contains an 'AUTOGENERATED' comment.
All indexes *before* the AUTOGENERATED comment will be written
back unchanged. All indexes *after* the AUTOGENERATED comment
will be updated with the latest query counts (query counts are
reset by --clear_datastore). Indexes that aren't yet in the file
will be appended to the AUTOGENERATED section.
We keep track of some data in order to avoid doing repetitive work:
- if index.yaml is fully manual, we keep track of its mtime to
avoid parsing it over and over;
- we keep track of the number of keys in the history dict since
the last time we updated index.yaml (or decided there was
nothing to update).
"""
index_yaml_file = os.path.join(self.root_path, 'index.yaml')
try:
index_yaml_mtime = os.path.getmtime(index_yaml_file)
except os.error:
index_yaml_mtime = None
index_yaml_changed = (index_yaml_mtime != self.index_yaml_mtime)
self.index_yaml_mtime = index_yaml_mtime
datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength(
)
history_changed = (query_ci_history_len != self.last_history_size)
self.last_history_size = query_ci_history_len
if not (index_yaml_changed or history_changed):
logging.debug('No need to update index.yaml')
return
if self.index_yaml_is_manual and not index_yaml_changed:
logging.debug('Will not update manual index.yaml')
return
if index_yaml_mtime is None:
index_yaml_data = None
else:
try:
fh = openfile(index_yaml_file, 'rU')
except IOError:
index_yaml_data = None
else:
try:
index_yaml_data = fh.read()
if not index_yaml_data:
raise EmptyIndexFileError(
'The index yaml file is empty. The file should at least have '
'an empty "indexes:" block.')
finally:
fh.close()
self.index_yaml_is_manual = (index_yaml_data is not None
and AUTO_MARKER not in index_yaml_data)
if self.index_yaml_is_manual:
logging.info('Detected manual index.yaml, will not update')
return
if index_yaml_data is None:
all_indexes = None
else:
try:
all_indexes = datastore_index.ParseIndexDefinitions(
index_yaml_data)
except yaml_errors.EventListenerError as e:
logging.error('Error parsing %s:\n%s', index_yaml_file, e)
return
except Exception as err:
logging.error('Error parsing %s:\n%s.%s: %s', index_yaml_file,
err.__class__.__module__, err.__class__.__name__,
err)
return
if index_yaml_data is None:
manual_part, prev_automatic_part = 'indexes:\n', ''
manual_indexes = None
else:
manual_part, prev_automatic_part = index_yaml_data.split(
AUTO_MARKER, 1)
if prev_automatic_part.startswith(AUTO_COMMENT):
prev_automatic_part = prev_automatic_part[len(AUTO_COMMENT):]
try:
manual_indexes = datastore_index.ParseIndexDefinitions(
manual_part)
except Exception as err:
logging.error('Error parsing manual part of %s: %s',
index_yaml_file, err)
return
automatic_part = GenerateIndexFromHistory(
datastore_stub.QueryHistory(), all_indexes, manual_indexes)
if (index_yaml_mtime is None and automatic_part == ''
or automatic_part == prev_automatic_part):
logging.debug('No need to update index.yaml')
return
try:
fh = openfile(index_yaml_file, 'w')
except IOError as err:
logging.error('Can\'t write index.yaml: %s', err)
return
try:
logging.info('Updating %s', index_yaml_file)
fh.write(manual_part)
fh.write(AUTO_MARKER)
fh.write(AUTO_COMMENT)
fh.write(automatic_part)
finally:
fh.close()
try:
self.index_yaml_mtime = os.path.getmtime(index_yaml_file)
except os.error as err:
logging.error('Can\'t stat index.yaml we just wrote: %s', err)
self.index_yaml_mtime = None
def _SetupIndexes(self, _open=open):
"""Ensure that the set of existing composite indexes matches index.yaml.
Create any new indexes, and delete indexes which are no longer required.
Args:
_open: Function used to open a file.
"""
if not self.__root_path:
logging.warning("No index.yaml was loaded.")
return
index_yaml_file = os.path.join(self.__root_path, 'index.yaml')
if (self.__cached_yaml[0] == index_yaml_file
and os.path.exists(index_yaml_file) and
os.path.getmtime(index_yaml_file) == self.__cached_yaml[1]):
requested_indexes = self.__cached_yaml[2]
else:
try:
index_yaml_mtime = os.path.getmtime(index_yaml_file)
fh = _open(index_yaml_file, 'r')
except (OSError, IOError):
logging.info("Error reading file")
index_yaml_data = None
else:
try:
index_yaml_data = fh.read()
finally:
fh.close()
requested_indexes = []
if index_yaml_data is not None:
index_defs = datastore_index.ParseIndexDefinitions(
index_yaml_data)
if index_defs is not None and index_defs.indexes is not None:
requested_indexes = datastore_index.IndexDefinitionsToProtos(
self.__app_id, index_defs.indexes)
self.__cached_yaml = (index_yaml_file, index_yaml_mtime,
requested_indexes)
existing_indexes = datastore_pb.CompositeIndices()
app_str = api_base_pb.StringProto()
app_str.set_value(self.__app_id)
self._Dynamic_GetIndices(app_str, existing_indexes)
requested = dict(
(x.definition().Encode(), x) for x in requested_indexes)
existing = dict((x.definition().Encode(), x)
for x in existing_indexes.index_list())
# Delete any indexes that are no longer requested.
deleted = 0
for key, index in existing.iteritems():
if key not in requested:
self._Dynamic_DeleteIndex(index, api_base_pb.VoidProto())
deleted += 1
# Add existing indexes in the index cache.
for key, index in existing.iteritems():
new_index = entity_pb.CompositeIndex()
new_index.CopyFrom(index)
ent_kind = new_index.definition().entity_type()
if ent_kind in self.__index_cache:
new_indexes = self.__index_cache[ent_kind]
new_indexes.append(new_index)
self.__index_cache[ent_kind] = new_indexes
else:
self.__index_cache[ent_kind] = [new_index]
# Compared the existing indexes to the requested ones and create any
# new indexes requested.
created = 0
for key, index in requested.iteritems():
if key not in existing:
new_index = entity_pb.CompositeIndex()
new_index.CopyFrom(index)
new_index.set_id(
self._Dynamic_CreateIndex(
new_index, api_base_pb.Integer64Proto()).value())
new_index.set_state(entity_pb.CompositeIndex.READ_WRITE)
self._Dynamic_UpdateIndex(new_index, api_base_pb.VoidProto())
created += 1
ent_kind = new_index.definition().entity_type()
if ent_kind in self.__index_cache:
new_indexes = self.__index_cache[ent_kind]
new_indexes.append(new_index)
self.__index_cache[ent_kind] = new_indexes
else:
self.__index_cache[ent_kind] = [new_index]
if created or deleted:
logging.info('Created %d and deleted %d index(es); total %d',
created, deleted, len(requested))
def SetupIndexes(app_id, root_path):
"""Ensure that the set of existing composite indexes matches index.yaml.
Note: this is similar to the algorithm used by the admin console for
the same purpose.
Args:
app_id: Application ID being served.
root_path: Path to the root of the application.
"""
index_yaml_file = os.path.join(root_path, 'index.yaml')
global _cached_yaml
if _cached_yaml[0] == index_yaml_file and os.path.exists(
index_yaml_file) and os.path.getmtime(
index_yaml_file) == _cached_yaml[1]:
requested_indexes = _cached_yaml[2]
else:
try:
index_yaml_mtime = os.path.getmtime(index_yaml_file)
fh = open(index_yaml_file, 'r')
except (OSError, IOError):
index_yaml_data = None
else:
try:
index_yaml_data = fh.read()
finally:
fh.close()
requested_indexes = []
if index_yaml_data is not None:
index_defs = datastore_index.ParseIndexDefinitions(index_yaml_data)
if index_defs is not None and index_defs.indexes is not None:
requested_indexes = datastore_index.IndexDefinitionsToProtos(
app_id, index_defs.indexes)
_cached_yaml = (index_yaml_file, index_yaml_mtime,
requested_indexes)
existing_indexes = datastore_admin.GetIndices(app_id)
requested = dict((x.definition().Encode(), x) for x in requested_indexes)
existing = dict((x.definition().Encode(), x) for x in existing_indexes)
created = 0
for key, index in requested.iteritems():
if key not in existing:
new_index = entity_pb.CompositeIndex()
new_index.CopyFrom(index)
id = datastore_admin.CreateIndex(new_index)
new_index.set_id(id)
new_index.set_state(entity_pb.CompositeIndex.READ_WRITE)
datastore_admin.UpdateIndex(new_index)
created += 1
deleted = 0
for key, index in existing.iteritems():
if key not in requested:
datastore_admin.DeleteIndex(index)
deleted += 1
if created or deleted:
logging.info("Created %d and deleted %d index(es); total %d", created,
deleted, len(requested))
