def __ResetPublishManager(self, server_url):
"""Resets publish manager.
Resets FdbService (unregisters FusionDbs/Portables registered for serving,
clears all existing Readers/Unpackers),
Cleans up publish info tables,
Sets cut specification into Fdb module,
Re-registers for serving published FusionDbs/Portables in Fdb
module,
Updates publish info in .htacccess-file.
Args:
server_url: server URL (scheme://host:port).
Raises:
psycopg2.Error/Warning, PublishServeException, ModFdbServeException.
"""
# Clear all Readers/Unpackers.
self._mod_fdb_serve_handler.ResetFdbService(server_url)
# Init cut specs.
self.__InitCutSpecs(server_url)
# Register all published targets for serving in Fdb module.
published_list = self._publish_helper.GetPublishInfoList()
for (target_path, host_name, db_name) in published_list:
(norm_db_name, db_type) = serve_utils.IdentifyPublishedDb(db_name)
if serve_utils.IsFusionDb(db_type):
gedb_path = self._publish_helper.BuildDbPublishPath(
host_name, norm_db_name)
target_gedb_path = self._publish_helper.BuildTargetPublishPath(
gedb_path, target_path)
else:
assert serve_utils.IsPortable(db_type)
target_gedb_path = norm_db_name
try:
self._mod_fdb_serve_handler.RegisterDatabaseForServing(
server_url, target_path, db_type, target_gedb_path)
except exceptions.PublishServeException as e:
# Unpublish the target path if registering for serving has failed.
logger.error(e)
self._publish_helper.DoUnpublish(target_path)
# Update .htaccess file.
self._publish_helper.UpdateHtaccessFile()
def HandleSyncRequest(self, request, response):
"""Handles database sync request.
Args:
request: request object.
response: response object.
Returns:
in response object, the list of files that need to be transfered.
Raises:
StreamPushServeException in case of invalid database.
"""
logger.debug("HandleSyncRequest..")
db_name = request.GetParameter(constants.DB_NAME)
if not db_name:
raise exceptions.StreamPushServeException(
"HandleSyncRequest: missing db name.")
(db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name)
if not serve_utils.IsFusionDb(db_type):
raise exceptions.StreamPushServeException(
"HandleSyncRequest: Unsupported DB type %s.", db_type)
client_host_name = request.GetClientHostName()
if not client_host_name:
raise exceptions.StreamPushServeException(
"HandleSyncRequest: missing Hostname.")
db_id = self.QueryDbId(client_host_name, db_path)
if db_id == 0:
raise exceptions.StreamPushServeException(
"HandleSyncRequest: Database %s is not registered on server." %
db_name)
transfer_file_paths = self.SynchronizeDb(db_id, db_type, client_host_name)
if not transfer_file_paths:
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
else:
for transfer_file_path in transfer_file_paths:
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_FILE_NAME, transfer_file_path)
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_UPLOAD_NEEDED)
def GetDbPublishPathPrefix(self, db_type, fusion_hostname):
"""Gets publish(push) path prefix for Fusion/Portable database.
Args:
db_type: database type.
fusion_hostname: Fusion hostname.
Returns:
database publish path prefix which is
for Fusion database - server_prefix/fusion_hostname, e.g
/gevol/published_dbs/stream_space/fusion_host_name
for Portable database - empty string.
"""
if serve_utils.IsFusionDb(db_type):
return self.GetFusionDbPublishPathPrefix(fusion_hostname)
elif serve_utils.IsPortable(db_type):
return ""
else:
raise exceptions.StreamPushServeException(
"HandleSyncRequest: Unsupported DB type %s.", db_type)
class StreamPushManager(stream_manager.StreamManager):
"""Class for handling stream data managing requests.
Stream data managing requests: AddDb, PushDb, DeleteDb, GarbageCollect.
"""
def __init__(self):
"""Inits StreamPushManager."""
super(StreamPushManager, self).__init__()
self._allow_symlinks = "N"
self._ReadPublishRootConfig()
def _ReadPublishRootConfig(self):
self._allow_symlinks = "N"
config_path = os.path.join(os.path.dirname(self.server_prefix), ".config")
try:
# Older publish roots may not have the .config file so we default to "N".
if os.path.exists(config_path):
sub_groups = utils.MatchPattern(
config_path,
"^AllowSymLinks:\\s*([YyNn])\\s*$")
if sub_groups:
self._allow_symlinks = sub_groups[0]
else:
raise exceptions.StreamPushServeException(
"Invalid Publish root config.")
finally:
logger.info("AllowSymlinks: " + self._allow_symlinks)
def HandleCleanupRequest(self, request, response):
"""Handles cleanup request.
Unregisters all portable globes that do not exist in file system.
Args:
request: request object.
response: response object.
Returns:
in response object list of portables that were unregistered/unpublished in
case of success, or list of portables that are registered but do not exist
in file system in case of failure.
"""
cmd = request.GetParameter(constants.CMD)
assert cmd and (cmd == constants.CMD_CLEANUP)
# Get information about registered portables.
query_string = "SELECT db_name FROM db_table WHERE host_name = ''"
results = self.DbQuery(query_string)
# Flag for whether globes directory is mounted and at least one portable
# globe exists. If not, don't remove Portables from postgres db.
is_globes_mounted = (
os.path.exists(stream_manager.StreamManager.CUTTER_GLOBES_PATH) and
serve_utils.ExistsPortableInDir(
stream_manager.StreamManager.CUTTER_GLOBES_PATH))
if not is_globes_mounted:
logger.warning(
"HandleCleanupRequest: No portable files in directory %s."
" Volume may not be mounted.",
stream_manager.StreamManager.CUTTER_GLOBES_PATH)
logger.warning("Cleanup has not been run.")
registered_portables = [
{"name": db_name,
"path": "{0}{1}".format(
stream_manager.StreamManager.CUTTER_GLOBES_PATH,
db_name)} for db_name in results]
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_DATA, json.dumps(registered_portables))
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_MESSAGE,
"No portable files in directory {0}."
" Volume may not be mounted.".format(
stream_manager.StreamManager.CUTTER_GLOBES_PATH))
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_FAILURE)
return
unregistered_portables = []
for db_name in results:
# Get database type, path.
(db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name)
assert serve_utils.IsPortable(db_type)
publish_db_path = "{0}{1}".format(
stream_manager.StreamManager.CUTTER_GLOBES_PATH, db_path)
if not os.path.exists(publish_db_path):
# Check if non-existing portable is registered.
client_host_name = ""
db_id = self.QueryDbId(client_host_name, db_path)
if db_id != 0:
# Unpublish portable.
self._UnpublishPortable(db_id)
# Unregister portable.
self._UnregisterPortable(db_id)
unregistered_portables.append(
{"name": db_name, "path": publish_db_path})
logger.info("Push info cleanup is complete.")
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_DATA, json.dumps(unregistered_portables))
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
def HandleQueryRequest(self, request, response):
"""Handles query requests.
Args:
request: request object.
response: response object.
Raises:
StreamPushServeException.
"""
query_cmd = request.GetParameter(constants.QUERY_CMD)
if not query_cmd:
raise exceptions.StreamPushServeException(
"Internal Error - Missing Query Command.")
# List all DBs registered on server.
if query_cmd == constants.QUERY_CMD_LIST_DBS:
query_string = "SELECT host_name, db_name, db_pretty_name FROM db_table"
results = self.DbQuery(query_string)
for line in results:
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_HOST_NAME, line[0])
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_DB_NAME, line[1])
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_DB_PRETTY_NAME, line[2])
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
# List all published DBs.
elif query_cmd == constants.QUERY_CMD_PUBLISHED_DBS:
query_string = """
SELECT target_table.target_path, virtual_host_table.virtual_host_name,
db_table.db_name, db_table.db_pretty_name, db_table.host_name
FROM db_table, virtual_host_table, target_db_table, target_table
WHERE db_table.db_id = target_db_table.db_id AND
target_table.target_id = target_db_table.target_id AND
virtual_host_table.virtual_host_id = target_db_table.virtual_host_id
"""
results = self.DbQuery(query_string)
for line in results:
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_TARGET_PATH, line[0])
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_VS_NAME, line[1])
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_DB_NAME, line[2])
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_DB_PRETTY_NAME, line[3])
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_HOST_NAME, line[4])
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
# Get DB details.
elif query_cmd == constants.QUERY_CMD_DB_DETAILS:
db_name = request.GetParameter(constants.DB_NAME)
if not db_name:
raise exceptions.StreamPushServeException(
"HandleQueryRequest: missing db name")
client_host_name = request.GetClientHostName()
if not client_host_name:
raise exceptions.StreamPushServeException(
"HandleQueryRequest: missing Hostname.")
query_string = (
"SELECT file_path FROM files_table WHERE file_id IN ("
"SELECT file_id FROM db_files_table WHERE db_id IN ("
"SELECT db_id FROM db_table WHERE "
"host_name = %s AND db_name = %s))")
results = self.DbQuery(query_string, (client_host_name, db_name))
for line in results:
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_FILE_NAME, line)
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
# Get server prefix - path to published_dbs directory.
elif query_cmd == constants.QUERY_CMD_SERVER_PREFIX:
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_SERVER_PREFIX, self.server_prefix)
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
# Get client hostname.
elif query_cmd == constants.QUERY_CMD_HOST_ROOT:
client_host_name = request.GetClientHostName()
if not client_host_name:
raise exceptions.StreamPushServeException(
"HandleQueryRequest: missing Hostname.")
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_HOST_ROOT, client_host_name)
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
# Get server hostname.
elif query_cmd == constants.QUERY_CMD_SERVER_HOST:
host_name = socket.gethostname()
full_host_name = socket.getfqdn()
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_SERVER_HOST, host_name)
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_SERVER_HOST_FULL, full_host_name)
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
# Get allow symbolic link parameter.
elif query_cmd == constants.QUERY_CMD_ALLOW_SYM_LINKS:
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_ALLOW_SYM_LINKS, self._allow_symlinks)
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
else:
raise exceptions.StreamPushServeException(
"Internal Error - Invalid Query Command: %s." % query_cmd)
def HandleAddDbRequest(self, request, response):
"""Handles add database request.
Args:
request: request object.
response: response object.
Raises:
StreamPushServeException.
"""
logger.debug("HandleAddDbRequest..")
db_name = request.GetParameter(constants.DB_NAME)
db_pretty_name = request.GetParameter(constants.DB_PRETTY_NAME)
if not db_name:
raise exceptions.StreamPushServeException(
"Internal Error - HandleAddDbRequest: missing db name")
if not db_pretty_name:
# if db_pretty_name is not specified, set it equal to db_name.
# Note: db_pretty_name is a database alias to make it easy to remember.
# For the fusion databases, we set it equal to the asset root database
# path, by default.
db_pretty_name = db_name
file_paths = request.GetMultiPartParameter(constants.FILE_PATH)
file_sizes = request.GetMultiPartParameter(constants.FILE_SIZE)
if (not file_sizes) or (not file_paths):
raise exceptions.StreamPushServeException(
"Internal Error - HandleAddDbRequest: missing file paths/sizes")
if len(file_paths) != len(file_sizes):
raise exceptions.StreamPushServeException(
"Internal Error - HandleAddDbRequest:"
" file_paths/file_sizes mismatch: %d/%d" % (len(file_paths),
len(file_sizes)))
db_timestamp = request.GetParameter(constants.DB_TIMESTAMP)
db_size = request.GetParameter(constants.DB_SIZE)
# Full ISO 8601 datetime format: '%Y-%m-%dT%H:%M:%S.%f-TZ'.
# Check if database timestamp begins with a valid date format.
if db_timestamp:
try:
datetime.datetime.strptime(db_timestamp[:10],
serve_utils.DATE_ISO_FORMAT)
except ValueError, e:
logger.warning(
"Database/Portable timestamp format is not valid. Error: %s", e)
db_timestamp = None
# Collecting database properties and packing into db_flags variable.
db_use_google_basemap = request.GetParameter(
constants.DB_USE_GOOGLE_BASEMAP)
logger.debug("HandleAddDbRequest: db_use_google_basemap: %s",
db_use_google_basemap)
db_flags = 0
if db_use_google_basemap and int(db_use_google_basemap) == 1:
db_flags = basic_types.DbFlags.USE_GOOGLE_BASEMAP
(db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name)
if serve_utils.IsFusionDb(db_type):
client_host_name = request.GetClientHostName()
if not client_host_name:
raise exceptions.StreamPushServeException(
"HandleAddDbRequest: missing Hostname.")
elif serve_utils.IsPortable(db_type):
# Note: The client host name is not used for portable globes, so we just
# make it an empty string.
client_host_name = ""
# Get timestamp and size from portable info.
db_info = basic_types.DbInfo()
db_info.name = db_name
db_info.type = db_type
serve_utils.GlxDetails(db_info)
db_timestamp = db_info.timestamp
db_size = db_info.size
else:
raise exceptions.StreamPushServeException(
"HandleAddDbRequest: Unsupported DB type %s.", db_type)
# Check if database already exists.
# The assumption is if it already exists, all the files and db-to-files
# links have already been published to the database.
db_id, curr_db_flags = self.QueryDbIdAndFlags(client_host_name, db_path)
logger.debug("HandleAddDbRequest: db_id: %d, curr_db_flags: %d",
db_id, curr_db_flags)
if db_id > 0:
# It's OK if the database already exists. Don't throw an exception.
# Note: db_table.db_flags field was added in GEE 5.0.1, so for databases
# pushed with GEE Fusion 5.0.0 and less we need to update db_flags in
# postgres.
if db_flags and (not curr_db_flags):
# Update db_table.db_flags for existing database.
query_string = ("UPDATE db_table SET db_flags = %s WHERE db_id = %s")
self.DbModify(query_string, (db_flags, db_id))
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
return
# We will now add the following entries into the db:
# 1) the db_table entry for the database
# 2) the files_table entries for each file path
# 3) the db_files_table entries linking each file to this database
# If anything fails in here, we want to remove the db_table entry (and any
# db_files_table entries) so that publish will start from this point on
# the next try.
try:
# Add the db entry.
query_string = (
"INSERT INTO db_table (host_name, db_name, db_pretty_name,"
" db_timestamp, db_size, db_flags) "
"VALUES(%s, %s, %s, %s, %s, %s)")
self.DbModify(query_string,
(client_host_name, db_name, db_pretty_name,
db_timestamp, db_size, db_flags))
# Record the db_id now, if it succeeded, and the following inserts fail,
# we can at least back some of the inserts out so we can try again later.
db_id = self.QueryDbId(client_host_name, db_name)
if db_id == 0:
raise exceptions.StreamPushServeException(
"HandleAddDbRequest: unable to add database: hostname:"
" %s, database name: %s" % (client_host_name, db_name))
# Check which files already exist in the files_table.
file_path_exists_flags = self._CheckFileExistence(client_host_name,
file_paths)
assert file_path_exists_flags
# Add files entries.
# Batch the SQL commands in groups of 1000 speeds things up noticeably.
# fewer gets slightly worse, more is not noticeable.
group_size = 1000
batcher = batch_sql_manager.BatchSqlManager(
self.stream_db, group_size, logger)
query_string = ("INSERT INTO files_table "
"(host_name, file_path, file_size, file_status) "
"VALUES(%s, %s, %s, 0)")
for i in range(len(file_paths)):
# Add only the newer files to avoid throwing SQLException.
if not file_path_exists_flags[i]:
batcher.AddModify(
query_string,
[client_host_name, file_paths[i], file_sizes[i]])
batcher.ExecuteRemaining()
# Finally update the db_files_table by first querying the file_ids from
# the files_table.
file_ids = self._QueryFileIds(client_host_name, file_paths)
assert file_ids
query_string = ("INSERT INTO db_files_table (db_id, file_id) "
"VALUES(%s, %s)")
parameters = [(db_id, file_id) for file_id in file_ids]
batcher.AddModifySet(query_string, parameters)
batcher.ExecuteRemaining()
batcher.Close() # Need to close to release SQL resources.
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
except Exception as e:
# Record the internal exception to pass out via the final exception.
error_message = ("HandleAddDbRequest:"
" Error adding files to publisher database\n") + repr(e)
# Clean up any remnants of this db_id, so that a subsequent attempt to
# publish will start over at this step.
# Note: this assumes that our connection is still valid.
try:
if db_id > 0:
# Note: this will leave behind some entries in files_table, but this
# will not hurt anything.
query_string = "DELETE FROM db_files_table WHERE db_id = %s"
self.DbModify(query_string, [db_id])
query_string = "DELETE FROM db_table WHERE db_id = %s"
self.DbModify(query_string, [db_id])
except exceptions.Error:
error_message += "\nAttempt to cleanup the database failed"
raise exceptions.StreamPushServeException(error_message)
def HandleDeleteDbRequest(self, request, response):
"""Handles delete database requests.
Args:
request: request object.
response: response object.
Raises:
OSError, psycopg2.Warning/Error, StreamPushServeException
"""
logger.debug("HandleDeleteDbRequest...")
db_name = request.GetParameter(constants.DB_NAME)
if not db_name:
raise exceptions.StreamPushServeException(
"HandleDeleteDbRequest: Missing database name.")
(db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name)
if serve_utils.IsFusionDb(db_type):
client_host_name = request.GetClientHostName()
if not client_host_name:
raise exceptions.StreamPushServeException(
"HandleDeleteDbRequest: missing Hostname.")
elif serve_utils.IsPortable(db_type):
# Note: The client host name is not used for portable globes, so we just
# make it an empty string.
client_host_name = ""
else:
raise exceptions.StreamPushServeException(
"HandleDeleteDbRequest: Unsupported DB type %s.", db_type)
# Check if the database exists.
db_id = self.QueryDbId(client_host_name, db_path)
if db_id == 0:
raise exceptions.StreamPushServeException(
"HandleDeleteDbRequest: Could not find database: "
"Fusion host: {0} Database name: {1}.".format(
client_host_name, db_path))
# Check if the database is currently published.
if self._QueryDbPublished(db_id):
raise exceptions.StreamPushServeException(
"HandleDeleteDbRequest: Database is currently published."
" Please unpublish it first.")
# Delete the entries in db_table.
query_string = "DELETE FROM db_table WHERE db_id = %s"
self.DbModify(query_string, (db_id,))
if serve_utils.IsFusionDb(db_type):
# Delete the entries in db_files_table. The entries in the
# files_table and the actual files are untouched. Those will be garbage
# collected at the request of the user.
query_string = "DELETE FROM db_files_table WHERE db_id = %s"
self.DbModify(query_string, [db_id])
elif serve_utils.IsPortable(db_type):
query_string = ("SELECT file_id FROM db_files_table WHERE db_id = %s")
rs_db_files = self.DbQuery(query_string, (db_id,))
if rs_db_files:
assert len(rs_db_files) == 1
file_id = rs_db_files[0]
# Delete the entries in db_files_table.
delete_db_files_table_cmd = (
"DELETE FROM db_files_table WHERE db_id = %s")
self.DbModify(delete_db_files_table_cmd, (db_id,))
# Get file_path from files_table and delete file.
query_string = ("SELECT file_path FROM files_table"
" WHERE file_id = %s")
rs_files = self.DbQuery(query_string, (file_id,))
if rs_files:
assert len(rs_files) == 1
assert db_path == rs_files[0]
if os.path.exists(rs_files[0]):
os.unlink(rs_files[0])
# Delete the file entry from files_table.
delete_files_table_cmd = "DELETE FROM files_table WHERE file_id = %s"
self.DbModify(delete_files_table_cmd, (file_id,))
else:
raise exceptions.StreamPushServeException(
"Unsupported DB type %s.", db_type)
http_io.ResponseWriter.AddBodyElement(
response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
def __GetPublishManifest(self, publish_def):
"""Build publish manifest files.
Publish manifest files: dbroot.v5p.DEFAULT, {earth,maps}.json, search.json,
serverdb.config, search.manifest.
Args:
publish_def: a PublishDef object encapsulating set of the publish
parameters.
"""
assert serve_utils.IsFusionDb(publish_def.db_type)
# Get publish URLs.
# Build stream URL based on Virtual Host URL.
vh_url, vh_ssl = self._publish_helper.QueryVh(
publish_def.virtual_host_name)
vh_base_url = self._publish_helper.GetVhBaseUrl(vh_url, vh_ssl)
stream_url = urlparse.urljoin(vh_base_url, publish_def.target_path)
logger.debug("Stream URL: %s", stream_url)
# Get database ID from gesearch database.
search_db_id = self._publish_helper.GetSearchDbId(
publish_def.client_host_name, publish_def.db_name)
# Build end_snippet proto section.
# Set end_snippet_proto to empty string - nothing to merge.
snippets_set = None
end_snippet_proto = None
if publish_def.snippets_set_name:
snippets_set = self._snippets_manager.GetSnippetSetDetails(
publish_def.snippets_set_name)
logger.debug("Snippets set: %s", snippets_set)
# Get list of search definitions.
search_def_list = self.__GetSearchDefs(
publish_def.search_tabs,
False, # is_supplemental
search_db_id,
publish_def.poi_federated,
publish_def.poi_suggestion)
# Get list of supplemental search definitions.
sup_search_def_list = self.__GetSearchDefs(
publish_def.sup_search_tabs,
True, # is_supplemental
search_db_id,
publish_def.poi_federated,
publish_def.poi_suggestion)
supplemental_ui_label = None
if sup_search_def_list:
if len(sup_search_def_list) == 1:
supplemental_ui_label = sup_search_def_list[0].label
else:
supplemental_ui_label = PublishManager.SUPPLEMENTAL_UI_LABEL_DEFAULT
if publish_def.supplemental_ui_label:
# Override with user specified value for the supplemental UI label.
supplemental_ui_label = publish_def.supplemental_ui_label
if publish_def.db_type == basic_types.DbType.TYPE_GE:
# Get end_snippet of protobuf dbroot - integrates snippets set and search
# services.
search_tab_id = None
if publish_def.need_search_tab_id:
# Using a target path as an ID for main search services by appending it
# to a search service label allows us to differentiate search tabs from
# different databases in Earth Client.
search_tab_id = publish_def.target_path[1:]
supplemental_search_url = PublishManager.SUPPLEMENTAL_SEARCH_URL
end_snippet_proto = dbroot_writer.CreateEndSnippetProto(
snippets_set, search_def_list, search_tab_id,
supplemental_ui_label, supplemental_search_url, logger)
# Note: useful for debugging.
# if __debug__:
# logger.debug("Proto end_snippet: %s", end_snippet_proto)
try:
# Get publish manifest.
publish_helper = libgepublishmanagerhelper.PublishManagerHelper(
logger)
publish_manifest = libgepublishmanagerhelper.ManifestEntryVector()
# Prepare publish config.
publish_config = libgepublishmanagerhelper.PublishConfig()
publish_config.fusion_host = publish_def.client_host_name
publish_config.db_path = publish_def.db_name
publish_config.stream_url = stream_url
publish_config.end_snippet_proto = (end_snippet_proto
if end_snippet_proto else "")
publish_config.server_prefix = self._publish_helper.server_prefix
publish_helper.GetPublishManifest(publish_config, publish_manifest)
logger.debug("PublishDatabase: publish manifest size %s.",
len(publish_manifest))
if not publish_manifest:
raise exceptions.PublishServeException(
"Unable to create publish manifest. Database is not pushed."
)
# Creates the search.json file and append it to publish manifest.
# Note: we append the search.json and supplemental search
# UI html files to the publish manifest if it is needed.
if sup_search_def_list:
search_def_list.extend(sup_search_def_list)
if search_def_list:
publish_tmp_dir_path = os.path.normpath(
publish_helper.TmpDir())
search_json_local_path = (
"%s/%s" %
(publish_tmp_dir_path, PublishManager.SEARCH_JSON_PATH))
self.__CreateSearchJsonFile(search_def_list,
search_json_local_path)
search_json_dbroot_path = PublishManager.SEARCH_JSON_PATH
publish_manifest.push_back(
libgepublishmanagerhelper.ManifestEntry(
search_json_dbroot_path, search_json_local_path))
# Append supplemental search UI html to publish manifest.
if ((publish_def.db_type == basic_types.DbType.TYPE_GE)
and supplemental_ui_label):
search_html_local_path = os.path.join(
PublishManager.HTDOCS_EARTH_PATH,
PublishManager.SUPPLEMENTAL_SEARCH_UI_HTML)
search_html_dbroot_path = PublishManager.SEARCH_HTML
publish_manifest.push_back(
libgepublishmanagerhelper.ManifestEntry(
search_html_dbroot_path, search_html_local_path))
# {gedb/,mapdb/} path is {server_prefix}/{fusion_host}{db_path}.
gedb_path = self._publish_helper.BuildDbPublishPath(
publish_def.client_host_name, publish_def.db_name)
# Transfer publish manifest files into published DBs directory.
db_path_prefix = self._publish_helper.BuildTargetPublishPath(
gedb_path, publish_def.target_path)
self._TransferPublishManifest(publish_manifest, db_path_prefix,
publish_def.force_copy)
except Exception:
# Delete target's publish directory in case of any error.
self._publish_helper.DeleteTargetPublishDir(
publish_def.target_path, publish_def.client_host_name,
publish_def.db_name)
raise
finally:
# Reset PublishManagerHelper processor (deletes publish temp. directory
# /tmp/publish.*).
publish_helper.Reset()
def __GetPublishParameters(self, request):
"""Gets and verifies all publish request parameters.
Args:
request: request object.
Returns:
The PublishDef object encapsulating set of the publish parameters.
"""
publish_def = PublishDef()
# Get Virtual Host name parameter.
publish_def.virtual_host_name = request.GetParameter(
constants.VIRTUAL_HOST_NAME)
if not publish_def.virtual_host_name:
raise exceptions.PublishServeException(
"Missing Virtual Host name.")
assert isinstance(publish_def.virtual_host_name, str)
# Get target path
target_path = request.GetParameter(constants.TARGET_PATH)
if not target_path:
raise exceptions.PublishServeException("Missing target path.")
assert isinstance(target_path, str)
# Normalize target path.
publish_def.target_path = serve_utils.NormalizeTargetPath(target_path)
if not publish_def.target_path:
raise exceptions.PublishServeException(
"Not valid target path %s (path format is /sub_path1[/sub_path2])."
% target_path)
self.__VerifyTargetPath(publish_def.target_path)
logger.debug("target path: %s", publish_def.target_path)
# Check whether target path is already in use.
if self._publish_helper.IsTargetPathUsed(publish_def.target_path):
raise exceptions.PublishServeException(
"Target path %s is already in use. Note that paths are "
"case insensitve. Input another path or"
" un-publish database using this path." %
publish_def.target_path)
# Get db name parameter.
publish_def.db_name = request.GetParameter(constants.DB_NAME)
if not publish_def.db_name:
raise exceptions.PublishServeException("Missing database name.")
logger.debug("db_name: %s", publish_def.db_name)
assert isinstance(publish_def.db_name, str)
(publish_def.db_name,
publish_def.db_type) = serve_utils.IdentifyPublishedDb(
publish_def.db_name)
if serve_utils.IsFusionDb(publish_def.db_type):
# Get host name parameter that is part of database ID (host, db_name).
publish_def.client_host_name = request.GetParameter(
constants.HOST_NAME)
if not publish_def.client_host_name:
raise exceptions.PublishServeException("Missing Hostname.")
assert isinstance(publish_def.client_host_name, str)
elif serve_utils.IsPortable(publish_def.db_type):
# There is no a client hostname for Portable databases.
publish_def.client_host_name = ""
else:
raise exceptions.PublishServeException("Unsupported DB type %s." %
publish_def.db_type)
# Get snippets set name parameter.
publish_def.snippets_set_name = request.GetParameter(
constants.SNIPPET_SET_NAME)
logger.debug(
"Snippets set name: %s ", publish_def.snippet_set_name
if publish_def.snippets_set_name else "not specified.")
# Get search definition name parameter.
publish_def.search_tabs = request.GetMultiPartParameter(
constants.SEARCH_DEF_NAME)
if publish_def.search_tabs:
logger.debug("Search tabs: %s", publish_def.search_tabs)
else:
logger.debug("Search tabs are not specified.")
# Get supplemental search definition name parameter.
publish_def.sup_search_tabs = request.GetMultiPartParameter(
constants.SUPPLEMENTAL_SEARCH_DEF_NAME)
if publish_def.sup_search_tabs:
logger.debug("Supplemental search tabs: %s",
publish_def.sup_search_tabs)
else:
logger.debug("Supplemental search tabs are not specified.")
publish_def.need_search_tab_id = request.GetBoolParameter(
constants.NEED_SEARCH_TAB_ID)
publish_def.poi_federated = request.GetBoolParameter(
constants.POI_FEDERATED)
publish_def.poi_suggestion = request.GetParameter(
constants.POI_SUGGESTION)
publish_def.supplemental_ui_label = request.GetParameter(
constants.SUPPLEMENTAL_UI_LABEL)
publish_def.serve_wms = request.GetBoolParameter(constants.SERVE_WMS)
publish_def.force_copy = request.IsForceCopy()
return publish_def
def HandlePublishRequest(self, request, response):
"""Handles publish database request.
Args:
request: request object.
response: response object.
Raises:
PublishServeException, psycopg2.Error/Warning.
"""
logger.debug("HandlePublishRequest...")
publish_def = self.__GetPublishParameters(request)
db_id = 0
# Handling Fusion database.
if serve_utils.IsFusionDb(publish_def.db_type):
# Check if the DB exists.
db_id = self._publish_helper.QueryDbId(
publish_def.client_host_name, publish_def.db_name)
if db_id == 0:
raise exceptions.PublishServeException(
"Database %s does not exist on server.\n"
"It needs to be registered/pushed before publishing." %
publish_def.db_name)
gedb_path = self._publish_helper.BuildDbPublishPath(
publish_def.client_host_name, publish_def.db_name)
logger.debug("GEDB path: %s, db type: %s", gedb_path,
publish_def.db_type)
target_gedb_path = self._publish_helper.BuildTargetPublishPath(
gedb_path, publish_def.target_path)
self.__GetPublishManifest(publish_def)
# Handling portable globe.
elif serve_utils.IsPortable(publish_def.db_type):
# Handling portable globes.
# Note: client host name is not used for portable globes.
assert (isinstance(publish_def.client_host_name, str)
and (not publish_def.client_host_name))
target_gedb_path = publish_def.db_name
db_id = self._publish_helper.QueryDbId(
publish_def.client_host_name, publish_def.db_name)
if db_id == 0:
raise exceptions.PublishServeException(
"Database %s does not exist (not registered/not pushed)." %
publish_def.db_name)
else:
raise exceptions.PublishServeException("Unsupported DB type %s." %
publish_def.db_type)
assert db_id != 0
try:
# Register FusionDb/Portable for serving in Fdb module.
request_url = request.GetParameter(constants.ORIGIN_REQUEST_HOST)
assert request_url
self._mod_fdb_serve_handler.RegisterDatabaseForServing(
request_url, publish_def.target_path, publish_def.db_type,
target_gedb_path)
except Exception:
self._publish_helper.DeleteTargetPublishDir(
publish_def.target_path, publish_def.client_host_name,
publish_def.db_name)
raise
try:
# Add entry to target_db_table.
self._publish_helper.HandlePublishRequest(db_id, publish_def)
# Update .htaccess file.
self._publish_helper.UpdateHtaccessFile()
http_io.ResponseWriter.AddBodyElement(response,
constants.HDR_STATUS_CODE,
constants.STATUS_SUCCESS)
except Exception:
self._publish_helper.DoUnpublish(publish_def.target_path)
self._mod_fdb_serve_handler.UnregisterDatabaseForServing(
request_url, publish_def.target_path)
raise
