def CreateStagingDirectory(self, tools_dir):
"""Creates a staging directory for uploading.
This is where we perform the necessary actions to create an application
directory for the update command to work properly - files are organized
into the static folder, and yaml files are generated where they can be
found later.
Args:
tools_dir: Path to the SDK tools directory
(typically .../google/appengine/tools)
Returns:
The path to a new temporary directory which contains generated yaml files
and a static file directory. For the most part, the rest of the update and
upload flow can resume identically to Python/PHP/Go applications.
Raises:
CompileError: if compilation of JSPs failed.
ConfigurationError: if the app to be staged has a configuration error.
IOError: if there was an I/O problem, for example when scanning jar files.
"""
stage_dir = tempfile.mkdtemp(prefix='appcfgpy')
static_dir = os.path.join(stage_dir, '__static__')
os.mkdir(static_dir)
self._CopyOrLink(self.basepath, stage_dir, static_dir, False)
self.app_engine_web_xml.app_root = stage_dir
if self.options.compile_jsps:
self._CompileJspsIfAny(tools_dir, stage_dir)
web_inf = os.path.join(stage_dir, 'WEB-INF')
web_inf_lib = os.path.join(web_inf, 'lib')
api_jar_dict = _FindApiJars(web_inf_lib)
api_versions = set(api_jar_dict.values())
if not api_versions:
api_version = None
elif len(api_versions) == 1:
api_version = api_versions.pop()
else:
raise ConfigurationError(
'API jars have inconsistent versions: %s' % api_jar_dict)
for staged_api_jar in api_jar_dict:
os.remove(staged_api_jar)
appengine_generated = os.path.join(stage_dir, 'WEB-INF',
'appengine-generated')
self._GenerateAppYaml(stage_dir, api_version, appengine_generated)
app_id = self.options.app_id or self.app_engine_web_xml.app_id
assert app_id, 'Missing app id'
for parser in self._XML_PARSERS:
xml_name = os.path.join(web_inf, parser.xml_name)
if os.path.exists(xml_name):
with open(xml_name) as xml_file:
xml_string = xml_file.read()
yaml_string = parser.xml_to_yaml_function(app_id, xml_string)
yaml_file = os.path.join(appengine_generated, parser.yaml_name)
with open(yaml_file, 'w') as yaml:
yaml.write(yaml_string)
indexes = []
for xml_name in ('datastore-indexes.xml',
os.path.join('appengine-generated',
'datastore-indexes-auto.xml')):
xml_name = os.path.join(self.basepath, 'WEB-INF', xml_name)
if os.path.exists(xml_name):
with open(xml_name) as xml_file:
xml_string = xml_file.read()
index_definitions = datastore_index_xml.IndexesXmlToIndexDefinitions(
xml_string)
indexes.extend(index_definitions.indexes)
if indexes:
yaml_string = datastore_index.IndexDefinitions(
indexes=indexes).ToYAML()
yaml_file = os.path.join(appengine_generated, 'index.yaml')
with open(yaml_file, 'w') as yaml:
yaml.write(yaml_string)
return stage_dir
if self.datastore_indexes_xml:
try:
manual_index_definitions = (
datastore_index_xml.IndexesXmlToIndexDefinitions(
self.datastore_indexes_xml))
except validation.ValidationError, e:
logging.error('Error parsing %s: %s',
datastore_indexes_xml_file, e)
return
else:
manual_index_definitions = datastore_index.IndexDefinitions(indexes=[])
if datastore_indexes_auto_xml:
try:
prev_auto_index_definitions = (
datastore_index_xml.IndexesXmlToIndexDefinitions(
datastore_indexes_auto_xml))
except validation.ValidationError, e:
logging.error('Error parsing %s: %s',
datastore_indexes_auto_xml_file, e)
return
else:
prev_auto_index_definitions = datastore_index.IndexDefinitions(indexes=[])
all_index_definitions = datastore_index.IndexDefinitions(
indexes=(manual_index_definitions.indexes +
prev_auto_index_definitions.indexes))
query_history = datastore_stub.QueryHistory()
auto_index_dict = GenerateIndexDictFromHistory(
query_history, all_index_definitions, manual_index_definitions)
auto_indexes, counts = self._IndexesFromIndexDict(auto_index_dict)
auto_index_definitions = datastore_index.IndexDefinitions(
class DatastoreIndexesAutoXmlUpdater(object):
"""Helper class for updating datastore-indexes-auto.xml.
This class maintains some state about the query history and the
datastore-indexes.xml and datastore-indexes-auto.xml files in order to
minimize the number of times datastore-indexes-auto.xml is rewritten.
"""
auto_generated = True
datastore_indexes_xml = None
datastore_indexes_xml_mtime = None
datastore_indexes_auto_xml_mtime = None
last_history_size = 0
def __init__(self, root_path):
self.root_path = root_path
def UpdateIndexConfig(self):
self.UpdateDatastoreIndexesAutoXml()
def UpdateDatastoreIndexesAutoXml(self, openfile=open):
"""Update datastore-indexes-auto.xml if appropriate."""
datastore_indexes_xml_file = os.path.join(
self.root_path, 'WEB-INF', 'datastore-indexes.xml')
try:
datastore_indexes_xml_mtime = os.path.getmtime(datastore_indexes_xml_file)
except os.error:
datastore_indexes_xml_mtime = None
if datastore_indexes_xml_mtime != self.datastore_indexes_xml_mtime:
self.datastore_indexes_xml_mtime = datastore_indexes_xml_mtime
if self.datastore_indexes_xml_mtime:
with openfile(datastore_indexes_xml_file) as f:
self.datastore_indexes_xml = f.read()
self.auto_generated = datastore_index_xml.IsAutoGenerated(
self.datastore_indexes_xml)
else:
self.auto_generated = True
self.datastore_indexes_xml = None
if not self.auto_generated:
logging.debug('Detected <datastore-indexes autoGenerated="false">,'
' will not update datastore-indexes-auto.xml')
return
datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength()
history_changed = (query_ci_history_len != self.last_history_size)
self.last_history_size = query_ci_history_len
if not history_changed:
logging.debug('No need to update datastore-indexes-auto.xml')
return
datastore_indexes_auto_xml_file = os.path.join(
self.root_path, 'WEB-INF', 'appengine-generated',
'datastore-indexes-auto.xml')
try:
with open(datastore_indexes_auto_xml_file) as f:
datastore_indexes_auto_xml = f.read()
except IOError, err:
datastore_indexes_auto_xml = None
if self.datastore_indexes_xml:
try:
manual_index_definitions = (
datastore_index_xml.IndexesXmlToIndexDefinitions(
self.datastore_indexes_xml))
except validation.ValidationError, e:
logging.error('Error parsing %s: %s',
datastore_indexes_xml_file, e)
return
def UpdateDatastoreIndexesAutoXml(self, openfile=open):
"""Update datastore-indexes-auto.xml if appropriate."""
datastore_indexes_xml_file = os.path.join(self.root_path, 'WEB-INF',
'datastore-indexes.xml')
try:
datastore_indexes_xml_mtime = os.path.getmtime(
datastore_indexes_xml_file)
except os.error:
datastore_indexes_xml_mtime = None
if datastore_indexes_xml_mtime != self.datastore_indexes_xml_mtime:
self.datastore_indexes_xml_mtime = datastore_indexes_xml_mtime
if self.datastore_indexes_xml_mtime:
with openfile(datastore_indexes_xml_file) as f:
self.datastore_indexes_xml = f.read()
self.auto_generated = datastore_index_xml.IsAutoGenerated(
self.datastore_indexes_xml)
else:
self.auto_generated = True
self.datastore_indexes_xml = None
if not self.auto_generated:
logging.debug('Detected <datastore-indexes autoGenerated="false">,'
' will not update datastore-indexes-auto.xml')
return
datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength(
)
history_changed = (query_ci_history_len != self.last_history_size)
self.last_history_size = query_ci_history_len
if not history_changed:
logging.debug('No need to update datastore-indexes-auto.xml')
return
datastore_indexes_auto_xml_file = os.path.join(
self.root_path, 'WEB-INF', 'appengine-generated',
'datastore-indexes-auto.xml')
try:
with open(datastore_indexes_auto_xml_file) as f:
datastore_indexes_auto_xml = f.read()
except IOError as err:
datastore_indexes_auto_xml = None
if self.datastore_indexes_xml:
try:
manual_index_definitions = (
datastore_index_xml.IndexesXmlToIndexDefinitions(
self.datastore_indexes_xml))
except validation.ValidationError as e:
logging.error('Error parsing %s: %s',
datastore_indexes_xml_file, e)
return
else:
manual_index_definitions = datastore_index.IndexDefinitions(
indexes=[])
if datastore_indexes_auto_xml:
try:
prev_auto_index_definitions = (
datastore_index_xml.IndexesXmlToIndexDefinitions(
datastore_indexes_auto_xml))
except validation.ValidationError as e:
logging.error('Error parsing %s: %s',
datastore_indexes_auto_xml_file, e)
return
else:
prev_auto_index_definitions = datastore_index.IndexDefinitions(
indexes=[])
all_index_definitions = datastore_index.IndexDefinitions(
indexes=(manual_index_definitions.indexes +
prev_auto_index_definitions.indexes))
query_history = datastore_stub.QueryHistory()
auto_index_dict = GenerateIndexDictFromHistory(
query_history, all_index_definitions, manual_index_definitions)
auto_indexes, counts = self._IndexesFromIndexDict(auto_index_dict)
auto_index_definitions = datastore_index.IndexDefinitions(
indexes=auto_indexes)
if auto_index_definitions == prev_auto_index_definitions:
return
try:
appengine_generated = os.path.dirname(
datastore_indexes_auto_xml_file)
if not os.path.exists(appengine_generated):
os.mkdir(appengine_generated)
with open(datastore_indexes_auto_xml_file, 'w') as f:
f.write(self._IndexXmlFromIndexes(auto_indexes, counts))
except os.error as err:
logging.error('Could not update %s: %s',
datastore_indexes_auto_xml_file, err)
