def CreateStagingDirectory(self, tools_dir): """Creates a staging directory for uploading. This is where we perform the necessary actions to create an application directory for the update command to work properly - files are organized into the static folder, and yaml files are generated where they can be found later. Args: tools_dir: Path to the SDK tools directory (typically .../google/appengine/tools) Returns: The path to a new temporary directory which contains generated yaml files and a static file directory. For the most part, the rest of the update and upload flow can resume identically to Python/PHP/Go applications. Raises: CompileError: if compilation of JSPs failed. ConfigurationError: if the app to be staged has a configuration error. IOError: if there was an I/O problem, for example when scanning jar files. """ stage_dir = tempfile.mkdtemp(prefix='appcfgpy') static_dir = os.path.join(stage_dir, '__static__') os.mkdir(static_dir) self._CopyOrLink(self.basepath, stage_dir, static_dir, False) self.app_engine_web_xml.app_root = stage_dir if self.options.compile_jsps: self._CompileJspsIfAny(tools_dir, stage_dir) web_inf = os.path.join(stage_dir, 'WEB-INF') web_inf_lib = os.path.join(web_inf, 'lib') api_jar_dict = _FindApiJars(web_inf_lib) api_versions = set(api_jar_dict.values()) if not api_versions: api_version = None elif len(api_versions) == 1: api_version = api_versions.pop() else: raise ConfigurationError( 'API jars have inconsistent versions: %s' % api_jar_dict) for staged_api_jar in api_jar_dict: os.remove(staged_api_jar) appengine_generated = os.path.join(stage_dir, 'WEB-INF', 'appengine-generated') self._GenerateAppYaml(stage_dir, api_version, appengine_generated) app_id = self.options.app_id or self.app_engine_web_xml.app_id assert app_id, 'Missing app id' for parser in self._XML_PARSERS: xml_name = os.path.join(web_inf, parser.xml_name) if os.path.exists(xml_name): with open(xml_name) as xml_file: xml_string = xml_file.read() yaml_string = parser.xml_to_yaml_function(app_id, xml_string) yaml_file = os.path.join(appengine_generated, parser.yaml_name) with open(yaml_file, 'w') as yaml: yaml.write(yaml_string) indexes = [] for xml_name in ('datastore-indexes.xml', os.path.join('appengine-generated', 'datastore-indexes-auto.xml')): xml_name = os.path.join(self.basepath, 'WEB-INF', xml_name) if os.path.exists(xml_name): with open(xml_name) as xml_file: xml_string = xml_file.read() index_definitions = datastore_index_xml.IndexesXmlToIndexDefinitions( xml_string) indexes.extend(index_definitions.indexes) if indexes: yaml_string = datastore_index.IndexDefinitions( indexes=indexes).ToYAML() yaml_file = os.path.join(appengine_generated, 'index.yaml') with open(yaml_file, 'w') as yaml: yaml.write(yaml_string) return stage_dir
if self.datastore_indexes_xml: try: manual_index_definitions = ( datastore_index_xml.IndexesXmlToIndexDefinitions( self.datastore_indexes_xml)) except validation.ValidationError, e: logging.error('Error parsing %s: %s', datastore_indexes_xml_file, e) return else: manual_index_definitions = datastore_index.IndexDefinitions(indexes=[]) if datastore_indexes_auto_xml: try: prev_auto_index_definitions = ( datastore_index_xml.IndexesXmlToIndexDefinitions( datastore_indexes_auto_xml)) except validation.ValidationError, e: logging.error('Error parsing %s: %s', datastore_indexes_auto_xml_file, e) return else: prev_auto_index_definitions = datastore_index.IndexDefinitions(indexes=[]) all_index_definitions = datastore_index.IndexDefinitions( indexes=(manual_index_definitions.indexes + prev_auto_index_definitions.indexes)) query_history = datastore_stub.QueryHistory() auto_index_dict = GenerateIndexDictFromHistory( query_history, all_index_definitions, manual_index_definitions) auto_indexes, counts = self._IndexesFromIndexDict(auto_index_dict) auto_index_definitions = datastore_index.IndexDefinitions(
class DatastoreIndexesAutoXmlUpdater(object): """Helper class for updating datastore-indexes-auto.xml. This class maintains some state about the query history and the datastore-indexes.xml and datastore-indexes-auto.xml files in order to minimize the number of times datastore-indexes-auto.xml is rewritten. """ auto_generated = True datastore_indexes_xml = None datastore_indexes_xml_mtime = None datastore_indexes_auto_xml_mtime = None last_history_size = 0 def __init__(self, root_path): self.root_path = root_path def UpdateIndexConfig(self): self.UpdateDatastoreIndexesAutoXml() def UpdateDatastoreIndexesAutoXml(self, openfile=open): """Update datastore-indexes-auto.xml if appropriate.""" datastore_indexes_xml_file = os.path.join( self.root_path, 'WEB-INF', 'datastore-indexes.xml') try: datastore_indexes_xml_mtime = os.path.getmtime(datastore_indexes_xml_file) except os.error: datastore_indexes_xml_mtime = None if datastore_indexes_xml_mtime != self.datastore_indexes_xml_mtime: self.datastore_indexes_xml_mtime = datastore_indexes_xml_mtime if self.datastore_indexes_xml_mtime: with openfile(datastore_indexes_xml_file) as f: self.datastore_indexes_xml = f.read() self.auto_generated = datastore_index_xml.IsAutoGenerated( self.datastore_indexes_xml) else: self.auto_generated = True self.datastore_indexes_xml = None if not self.auto_generated: logging.debug('Detected <datastore-indexes autoGenerated="false">,' ' will not update datastore-indexes-auto.xml') return datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength() history_changed = (query_ci_history_len != self.last_history_size) self.last_history_size = query_ci_history_len if not history_changed: logging.debug('No need to update datastore-indexes-auto.xml') return datastore_indexes_auto_xml_file = os.path.join( self.root_path, 'WEB-INF', 'appengine-generated', 'datastore-indexes-auto.xml') try: with open(datastore_indexes_auto_xml_file) as f: datastore_indexes_auto_xml = f.read() except IOError, err: datastore_indexes_auto_xml = None if self.datastore_indexes_xml: try: manual_index_definitions = ( datastore_index_xml.IndexesXmlToIndexDefinitions( self.datastore_indexes_xml)) except validation.ValidationError, e: logging.error('Error parsing %s: %s', datastore_indexes_xml_file, e) return
def UpdateDatastoreIndexesAutoXml(self, openfile=open): """Update datastore-indexes-auto.xml if appropriate.""" datastore_indexes_xml_file = os.path.join(self.root_path, 'WEB-INF', 'datastore-indexes.xml') try: datastore_indexes_xml_mtime = os.path.getmtime( datastore_indexes_xml_file) except os.error: datastore_indexes_xml_mtime = None if datastore_indexes_xml_mtime != self.datastore_indexes_xml_mtime: self.datastore_indexes_xml_mtime = datastore_indexes_xml_mtime if self.datastore_indexes_xml_mtime: with openfile(datastore_indexes_xml_file) as f: self.datastore_indexes_xml = f.read() self.auto_generated = datastore_index_xml.IsAutoGenerated( self.datastore_indexes_xml) else: self.auto_generated = True self.datastore_indexes_xml = None if not self.auto_generated: logging.debug('Detected <datastore-indexes autoGenerated="false">,' ' will not update datastore-indexes-auto.xml') return datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength( ) history_changed = (query_ci_history_len != self.last_history_size) self.last_history_size = query_ci_history_len if not history_changed: logging.debug('No need to update datastore-indexes-auto.xml') return datastore_indexes_auto_xml_file = os.path.join( self.root_path, 'WEB-INF', 'appengine-generated', 'datastore-indexes-auto.xml') try: with open(datastore_indexes_auto_xml_file) as f: datastore_indexes_auto_xml = f.read() except IOError as err: datastore_indexes_auto_xml = None if self.datastore_indexes_xml: try: manual_index_definitions = ( datastore_index_xml.IndexesXmlToIndexDefinitions( self.datastore_indexes_xml)) except validation.ValidationError as e: logging.error('Error parsing %s: %s', datastore_indexes_xml_file, e) return else: manual_index_definitions = datastore_index.IndexDefinitions( indexes=[]) if datastore_indexes_auto_xml: try: prev_auto_index_definitions = ( datastore_index_xml.IndexesXmlToIndexDefinitions( datastore_indexes_auto_xml)) except validation.ValidationError as e: logging.error('Error parsing %s: %s', datastore_indexes_auto_xml_file, e) return else: prev_auto_index_definitions = datastore_index.IndexDefinitions( indexes=[]) all_index_definitions = datastore_index.IndexDefinitions( indexes=(manual_index_definitions.indexes + prev_auto_index_definitions.indexes)) query_history = datastore_stub.QueryHistory() auto_index_dict = GenerateIndexDictFromHistory( query_history, all_index_definitions, manual_index_definitions) auto_indexes, counts = self._IndexesFromIndexDict(auto_index_dict) auto_index_definitions = datastore_index.IndexDefinitions( indexes=auto_indexes) if auto_index_definitions == prev_auto_index_definitions: return try: appengine_generated = os.path.dirname( datastore_indexes_auto_xml_file) if not os.path.exists(appengine_generated): os.mkdir(appengine_generated) with open(datastore_indexes_auto_xml_file, 'w') as f: f.write(self._IndexXmlFromIndexes(auto_indexes, counts)) except os.error as err: logging.error('Could not update %s: %s', datastore_indexes_auto_xml_file, err)