def HandleLocalTransferRequest(self, request, response): """Handles Local Transferring request. Args: request: request object. response: response object. Raises: StreamPushServeException """ logger.debug("HandleLocalTransferRequest...") src_path = request.GetParameter(constants.FILE_PATH) dest_path = request.GetParameter(constants.DEST_FILE_PATH) if (not src_path) or (not dest_path): raise exceptions.StreamPushServeException( "HandleLocalTransferRequest: Missing src/dest paths.") src_path = os.path.normpath(src_path) dest_path = os.path.normpath(dest_path) logger.debug("HandleLocalTransferRequest: %s to %s", src_path, dest_path) force_copy = request.IsForceCopy() prefer_copy = request.IsPreferCopy() if serve_utils.LocalTransfer(src_path, dest_path, force_copy, prefer_copy, self._allow_symlinks): http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) else: raise exceptions.StreamPushServeException("Local transfer failed.")
def HandleAddDbRequest(self, request, response): """Handles add database request. Args: request: request object. response: response object. Raises: StreamPushServeException. """ logger.debug("HandleAddDbRequest..") db_name = request.GetParameter(constants.DB_NAME) db_pretty_name = request.GetParameter(constants.DB_PRETTY_NAME) if not db_name: raise exceptions.StreamPushServeException( "Internal Error - HandleAddDbRequest: missing db name") if not db_pretty_name: # if db_pretty_name is not specified, set it equal to db_name. # Note: db_pretty_name is a database alias to make it easy to remember. # For the fusion databases, we set it equal to the asset root database # path, by default. db_pretty_name = db_name file_paths = request.GetMultiPartParameter(constants.FILE_PATH) file_sizes = request.GetMultiPartParameter(constants.FILE_SIZE) if (not file_sizes) or (not file_paths): raise exceptions.StreamPushServeException( "Internal Error - HandleAddDbRequest: missing file paths/sizes") if len(file_paths) != len(file_sizes): raise exceptions.StreamPushServeException( "Internal Error - HandleAddDbRequest:" " file_paths/file_sizes mismatch: %d/%d" % (len(file_paths), len(file_sizes))) db_timestamp = request.GetParameter(constants.DB_TIMESTAMP) db_size = request.GetParameter(constants.DB_SIZE) # Full ISO 8601 datetime format: '%Y-%m-%dT%H:%M:%S.%f-TZ'. # Check if database timestamp begins with a valid date format. if db_timestamp: try: datetime.datetime.strptime(db_timestamp[:10], serve_utils.DATE_ISO_FORMAT) except ValueError, e: logger.warning( "Database/Portable timestamp format is not valid. Error: %s", e) db_timestamp = None
def HandleSyncRequest(self, request, response): """Handles database sync request. Args: request: request object. response: response object. Returns: in response object, the list of files that need to be transfered. Raises: StreamPushServeException in case of invalid database. """ logger.debug("HandleSyncRequest..") db_name = request.GetParameter(constants.DB_NAME) if not db_name: raise exceptions.StreamPushServeException( "HandleSyncRequest: missing db name.") (db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name) if not serve_utils.IsFusionDb(db_type): raise exceptions.StreamPushServeException( "HandleSyncRequest: Unsupported DB type %s.", db_type) client_host_name = request.GetClientHostName() if not client_host_name: raise exceptions.StreamPushServeException( "HandleSyncRequest: missing Hostname.") db_id = self.QueryDbId(client_host_name, db_path) if db_id == 0: raise exceptions.StreamPushServeException( "HandleSyncRequest: Database %s is not registered on server." % db_name) transfer_file_paths = self.SynchronizeDb(db_id, db_type, client_host_name) if not transfer_file_paths: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) else: for transfer_file_path in transfer_file_paths: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_FILE_NAME, transfer_file_path) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_UPLOAD_NEEDED)
def HandleUnregisterPortableRequest(self, request, response): """Handles unregister portable globe requests. Cleans up table's entries related to specified portable globe. Args: request: request object. response: response object. Raises: OSError, psycopg2.Warning/Error, StreamPushServeException """ logger.debug("HandleUnregisterPortableRequest...") db_name = request.GetParameter(constants.DB_NAME) if not db_name: raise exceptions.StreamPushServeException( "HandleUnregisterPortableRequest: Missing database name.") (db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name) if not serve_utils.IsPortable(db_type): raise exceptions.StreamPushServeException( "HandleUnregisterPortableRequest: Unsupported DB type %s.", db_type) # Note: The client host name is not used for portable globes, so we just # make it an empty string. client_host_name = "" # Check if the database exists. db_id = self.QueryDbId(client_host_name, db_path) if db_id == 0: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) return # Check if the database is currently published. if self._QueryDbPublished(db_id): raise exceptions.StreamPushServeException( "HandleUnregisterPortableRequest: Database is currently published." " Please unpublish it first.") self._UnregisterPortable(db_id) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
def _QueryFileIds(self, client_host_name, file_paths): """Gets file's IDs by file path from files_table. Args: client_host_name: client host name. file_paths: a list of file paths. Returns: an array with an ID entry for the corresponding file_paths entries in the files_table or None in case the file_paths is empty list. Raises: StreamPushServeException in case the requested file path is not registered in the files_table. """ if not file_paths: return None not_valid_id = 0 fileid_list = [not_valid_id] * len(file_paths) # The "IN" call in SQL does not perform well enough to use a single # SQL statement here, but # in this case N/10 SQL calls using 10 IN is faster than N SQL calls. # Note: at 1000, the "IN" operation is unusable and 100 is roughly # equivalent to N single SQL queries. group_size = 10 cur_idx = 0 remaining = len(file_paths) groups = int(math.ceil(float(remaining) / float(group_size))) # Loop through the groups of group_size- paths and check their existence. for unused_i in xrange(groups): count = min(group_size, remaining) file_paths_temp = file_paths[cur_idx : cur_idx + count] assert file_paths_temp # Make a query to get a list of (file_path, file_id) tuples by # (hostname, file_path). # All files should have been registered in postgres . path_id_pairs = self._QueryFileIdsBatch(client_host_name, file_paths_temp) if (not path_id_pairs) or (len(path_id_pairs) != len(file_paths_temp)): raise exceptions.StreamPushServeException( "The publisher (gestream) database is not consistent.\n" "Not all Fusion database files are registered in files_table.") # Convert the result to a map<file_path, file_id>. path2id_map = dict(path_id_pairs) for j, file_path_j in enumerate(file_paths_temp): assert file_path_j in path2id_map fileid_list[cur_idx + j] = path2id_map[file_path_j] cur_idx += count remaining -= count return fileid_list
def _ReadPublishRootConfig(self): self._allow_symlinks = "N" config_path = os.path.join(os.path.dirname(self.server_prefix), ".config") try: # Older publish roots may not have the .config file so we default to "N". if os.path.exists(config_path): sub_groups = utils.MatchPattern( config_path, "^AllowSymLinks:\\s*([YyNn])\\s*$") if sub_groups: self._allow_symlinks = sub_groups[0] else: raise exceptions.StreamPushServeException( "Invalid Publish root config.") finally: logger.info("AllowSymlinks: " + self._allow_symlinks)
def GetDbPublishPathPrefix(self, db_type, fusion_hostname): """Gets publish(push) path prefix for Fusion/Portable database. Args: db_type: database type. fusion_hostname: Fusion hostname. Returns: database publish path prefix which is for Fusion database - server_prefix/fusion_hostname, e.g /gevol/published_dbs/stream_space/fusion_host_name for Portable database - empty string. """ if serve_utils.IsFusionDb(db_type): return self.GetFusionDbPublishPathPrefix(fusion_hostname) elif serve_utils.IsPortable(db_type): return "" else: raise exceptions.StreamPushServeException( "HandleSyncRequest: Unsupported DB type %s.", db_type)
def DoGet(self, request, response): """Handles GET request by delegating it to stream push manager. Args: request: request object. response: response object. Raises: StreamPushServeException, psycopg2.Warning/Error Returns: response object. """ # Check for init failure and return an error status and appropriate message. if not self._stream_push_manager: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_MESSAGE, ("Server-side Internal Error: Failure to init StreamPusher")) http_io.ResponseWriter.AddBodyElement(response, constants.HDR_STATUS_CODE, constants.STATUS_FAILURE) return try: cmd = request.GetParameter(constants.CMD) if not cmd: raise exceptions.StreamPushServeException( "Internal Error - Missing Request Command.") # Handle different query requests: ListDbs, PublishedDbs, Dbdetails... if cmd == constants.CMD_QUERY: self._stream_push_manager.HandleQueryRequest(request, response) # Ping gestream database. elif cmd == constants.CMD_PING: self._stream_push_manager.HandlePingRequest(request, response) # Register Fusion database/portable globe. elif cmd == constants.CMD_ADD_DB: self._stream_push_manager.HandleAddDbRequest(request, response) # Unregister Fusion database/ Delete portable globe. # TODO: consider to just unregister portable globe here, # while delete it in garbage collect. elif cmd == constants.CMD_DELETE_DB: self._stream_push_manager.HandleDeleteDbRequest(request, response) # Unregister portable globe. elif cmd == constants.CMD_CLEANUP_DB: self._stream_push_manager.HandleUnregisterPortableRequest( request, response) # Handle sync database request. elif cmd == constants.CMD_SYNC_DB: self._stream_push_manager.HandleSyncRequest(request, response) # Handle local transfer request. elif cmd == constants.CMD_LOCAL_TRANSFER: self._stream_push_manager.HandleLocalTransferRequest(request, response) # Handle garbage collect request. elif cmd == constants.CMD_GARBAGE_COLLECT: self._stream_push_manager.HandleGarbageCollectRequest(request, response) # Handle cleanup request. elif cmd == constants.CMD_CLEANUP: self._stream_push_manager.HandleCleanupRequest(request, response) else: raise exceptions.StreamPushServeException( "Internal Error - Invalid Request Command: %s." % cmd) return except exceptions.StreamPushServeException as e: logger.error(e) http_io.ResponseWriter.AddBodyElement(response, constants.HDR_STATUS_MESSAGE, e) except psycopg2.Warning as w: logger.error(w) http_io.ResponseWriter.AddBodyElement(response, constants.HDR_STATUS_MESSAGE, w) except psycopg2.Error as e: logger.error(e) http_io.ResponseWriter.AddBodyElement(response, constants.HDR_STATUS_MESSAGE, e) except Exception as e: logger.error(e) http_io.ResponseWriter.AddBodyElement(response, constants.HDR_STATUS_MESSAGE, "Server-side Internal Error") http_io.ResponseWriter.AddBodyElement(response, constants.HDR_STATUS_CODE, constants.STATUS_FAILURE)
def HandleGarbageCollectRequest(self, request, response): """Handles Garbage Collect request. Removes all unnecessary files and file references from the publish root volume and database. Args: request: request object. response: response object. Raises: psycopg2.Error/Warning """ logger.debug("HandleGarbageCollectRequest..") assert request.GetParameter(constants.CMD) == constants.CMD_GARBAGE_COLLECT parent_dirs_set = set() delete_count = 0 delete_size = 0 # The "NOT IN" sql operator is painfully slow for large collections of # files. # Instead we do 2 queries of all the file_id(s) in files_table and # db_files_table # and walk through the list removing files_table entries that don't occur # in db_files_table and vice versa. query = ("SELECT file_id, host_name, file_path, file_size" " FROM files_table ORDER BY file_id") rs_files = self.DbQuery(query) query = ("SELECT file_id FROM db_files_table" " GROUP BY file_id ORDER BY file_id") rs_db_files = self.DbQuery(query) db_file_id = sys.maxint if rs_db_files: db_file_iter = iter(rs_db_files) db_file_id = int(db_file_iter.next()) # Use a BatchSqlManager to batch up many individual SQL commands into # one postgres invocation. batcher = batch_sql_manager.BatchSqlManager( self.stream_db, 1000, logger) top_level_dir_prefix = self.server_prefix + "/" delete_files_table_cmd = "DELETE FROM files_table WHERE file_id = %s" delete_db_files_table_cmd = ( "DELETE FROM db_files_table WHERE file_id = %s") # We have 2 sorted lists of file_id's in the query results. # db_files_table is the subset of files_table # We will walk the two lists removing any files_table entries that don't # appear in db_files_table. for rs_file in rs_files: file_id = int(rs_file[0]) # Check the boundary case...this could happen and would basically mean # pain for whomever this happens to. if file_id == sys.maxint: logger.error("HandleGarbageCollectRequest has encountered a file_id " "equal to the max int value. " "The database has run out of valid file id's.") raise exceptions.StreamPushServeException( "The publisher database has run out of valid file id's." " This published database must likely be recreated from scratch.") if file_id < db_file_id: # Delete this file: # the files_table entry does not exist in the db_files_table # Check if the file exists and if so delete it. top_level_dir = top_level_dir_prefix + rs_file[1] server_file_path = os.path.normpath(top_level_dir + rs_file[2]) if os.path.exists(server_file_path): # Keep a set of parent dirs so that we can delete them later if # empty. parent_dir = os.path.dirname(server_file_path) while parent_dir != top_level_dir: parent_dirs_set.add(parent_dir) parent_dir = os.path.dirname(parent_dir) # Now we can delete the file from the file system. And, whether # it is successful, we delete the file entry from files_table. try: os.remove(server_file_path) delete_count += 1 delete_size += rs_file[3] # Delete the file entry from files_table. batcher.AddModify(delete_files_table_cmd, [file_id]) except OSError: logger.error( "HandleGarbageCollectRequest: Could not delete file %s.", server_file_path) else: # File does not exist - just delete the file entry from files_table. batcher.AddModify(delete_files_table_cmd, [file_id]) elif file_id == db_file_id: # This file is ok, skip to the next db_file_id try: db_file_id = int(db_file_iter.next()) except StopIteration: db_file_id = sys.maxint else: # Note: db_files_table's file_id(s) should be a subset of # files_table's file_id(s), if we encounter a db_files_table entry # which has no files_table entry, we will delete the db_files_table # entry. This really shouldn't happen, but who knows if a user pokes # around the psql database and deletes something they shouldn't. while db_file_id < file_id: batcher.AddModify(delete_db_files_table_cmd, [db_file_id]) try: db_file_id = int(db_file_iter.next()) except StopIteration: db_file_id = sys.maxint # Close up the sql objects and execute the remaining batch commands. batcher.ExecuteRemaining() batcher.Close() # Finally delete all the dirs if they are empty. We walk the list in the # reverse order coz we want to delete the child dirs before we try to # delete the parent dirs. parent_dirs_list = sorted(parent_dirs_set) for parent_dir in reversed(parent_dirs_list): try: os.rmdir(parent_dir) logger.debug("HandleGarbageCollectRequest: remove dir: %s", parent_dir) except OSError: pass # Queue up the stats for the garbage collection. http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DELETE_COUNT, delete_count) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DELETE_SIZE, delete_size) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) logger.debug("HandleGarbageCollectRequest done.")
class StreamPushManager(stream_manager.StreamManager): """Class for handling stream data managing requests. Stream data managing requests: AddDb, PushDb, DeleteDb, GarbageCollect. """ def __init__(self): """Inits StreamPushManager.""" super(StreamPushManager, self).__init__() self._allow_symlinks = "N" self._ReadPublishRootConfig() def _ReadPublishRootConfig(self): self._allow_symlinks = "N" config_path = os.path.join(os.path.dirname(self.server_prefix), ".config") try: # Older publish roots may not have the .config file so we default to "N". if os.path.exists(config_path): sub_groups = utils.MatchPattern( config_path, "^AllowSymLinks:\\s*([YyNn])\\s*$") if sub_groups: self._allow_symlinks = sub_groups[0] else: raise exceptions.StreamPushServeException( "Invalid Publish root config.") finally: logger.info("AllowSymlinks: " + self._allow_symlinks) def HandleCleanupRequest(self, request, response): """Handles cleanup request. Unregisters all portable globes that do not exist in file system. Args: request: request object. response: response object. Returns: in response object list of portables that were unregistered/unpublished in case of success, or list of portables that are registered but do not exist in file system in case of failure. """ cmd = request.GetParameter(constants.CMD) assert cmd and (cmd == constants.CMD_CLEANUP) # Get information about registered portables. query_string = "SELECT db_name FROM db_table WHERE host_name = ''" results = self.DbQuery(query_string) # Flag for whether globes directory is mounted and at least one portable # globe exists. If not, don't remove Portables from postgres db. is_globes_mounted = ( os.path.exists(stream_manager.StreamManager.CUTTER_GLOBES_PATH) and serve_utils.ExistsPortableInDir( stream_manager.StreamManager.CUTTER_GLOBES_PATH)) if not is_globes_mounted: logger.warning( "HandleCleanupRequest: No portable files in directory %s." " Volume may not be mounted.", stream_manager.StreamManager.CUTTER_GLOBES_PATH) logger.warning("Cleanup has not been run.") registered_portables = [ {"name": db_name, "path": "{0}{1}".format( stream_manager.StreamManager.CUTTER_GLOBES_PATH, db_name)} for db_name in results] http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DATA, json.dumps(registered_portables)) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_MESSAGE, "No portable files in directory {0}." " Volume may not be mounted.".format( stream_manager.StreamManager.CUTTER_GLOBES_PATH)) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_FAILURE) return unregistered_portables = [] for db_name in results: # Get database type, path. (db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name) assert serve_utils.IsPortable(db_type) publish_db_path = "{0}{1}".format( stream_manager.StreamManager.CUTTER_GLOBES_PATH, db_path) if not os.path.exists(publish_db_path): # Check if non-existing portable is registered. client_host_name = "" db_id = self.QueryDbId(client_host_name, db_path) if db_id != 0: # Unpublish portable. self._UnpublishPortable(db_id) # Unregister portable. self._UnregisterPortable(db_id) unregistered_portables.append( {"name": db_name, "path": publish_db_path}) logger.info("Push info cleanup is complete.") http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DATA, json.dumps(unregistered_portables)) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) def HandleQueryRequest(self, request, response): """Handles query requests. Args: request: request object. response: response object. Raises: StreamPushServeException. """ query_cmd = request.GetParameter(constants.QUERY_CMD) if not query_cmd: raise exceptions.StreamPushServeException( "Internal Error - Missing Query Command.") # List all DBs registered on server. if query_cmd == constants.QUERY_CMD_LIST_DBS: query_string = "SELECT host_name, db_name, db_pretty_name FROM db_table" results = self.DbQuery(query_string) for line in results: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_HOST_NAME, line[0]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DB_NAME, line[1]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DB_PRETTY_NAME, line[2]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # List all published DBs. elif query_cmd == constants.QUERY_CMD_PUBLISHED_DBS: query_string = """ SELECT target_table.target_path, virtual_host_table.virtual_host_name, db_table.db_name, db_table.db_pretty_name, db_table.host_name FROM db_table, virtual_host_table, target_db_table, target_table WHERE db_table.db_id = target_db_table.db_id AND target_table.target_id = target_db_table.target_id AND virtual_host_table.virtual_host_id = target_db_table.virtual_host_id """ results = self.DbQuery(query_string) for line in results: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_TARGET_PATH, line[0]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_VS_NAME, line[1]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DB_NAME, line[2]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DB_PRETTY_NAME, line[3]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_HOST_NAME, line[4]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get DB details. elif query_cmd == constants.QUERY_CMD_DB_DETAILS: db_name = request.GetParameter(constants.DB_NAME) if not db_name: raise exceptions.StreamPushServeException( "HandleQueryRequest: missing db name") client_host_name = request.GetClientHostName() if not client_host_name: raise exceptions.StreamPushServeException( "HandleQueryRequest: missing Hostname.") query_string = ( "SELECT file_path FROM files_table WHERE file_id IN (" "SELECT file_id FROM db_files_table WHERE db_id IN (" "SELECT db_id FROM db_table WHERE " "host_name = %s AND db_name = %s))") results = self.DbQuery(query_string, (client_host_name, db_name)) for line in results: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_FILE_NAME, line) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get server prefix - path to published_dbs directory. elif query_cmd == constants.QUERY_CMD_SERVER_PREFIX: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_SERVER_PREFIX, self.server_prefix) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get client hostname. elif query_cmd == constants.QUERY_CMD_HOST_ROOT: client_host_name = request.GetClientHostName() if not client_host_name: raise exceptions.StreamPushServeException( "HandleQueryRequest: missing Hostname.") http_io.ResponseWriter.AddBodyElement( response, constants.HDR_HOST_ROOT, client_host_name) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get server hostname. elif query_cmd == constants.QUERY_CMD_SERVER_HOST: host_name = socket.gethostname() full_host_name = socket.getfqdn() http_io.ResponseWriter.AddBodyElement( response, constants.HDR_SERVER_HOST, host_name) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_SERVER_HOST_FULL, full_host_name) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get allow symbolic link parameter. elif query_cmd == constants.QUERY_CMD_ALLOW_SYM_LINKS: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_ALLOW_SYM_LINKS, self._allow_symlinks) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) else: raise exceptions.StreamPushServeException( "Internal Error - Invalid Query Command: %s." % query_cmd) def HandleAddDbRequest(self, request, response): """Handles add database request. Args: request: request object. response: response object. Raises: StreamPushServeException. """ logger.debug("HandleAddDbRequest..") db_name = request.GetParameter(constants.DB_NAME) db_pretty_name = request.GetParameter(constants.DB_PRETTY_NAME) if not db_name: raise exceptions.StreamPushServeException( "Internal Error - HandleAddDbRequest: missing db name") if not db_pretty_name: # if db_pretty_name is not specified, set it equal to db_name. # Note: db_pretty_name is a database alias to make it easy to remember. # For the fusion databases, we set it equal to the asset root database # path, by default. db_pretty_name = db_name file_paths = request.GetMultiPartParameter(constants.FILE_PATH) file_sizes = request.GetMultiPartParameter(constants.FILE_SIZE) if (not file_sizes) or (not file_paths): raise exceptions.StreamPushServeException( "Internal Error - HandleAddDbRequest: missing file paths/sizes") if len(file_paths) != len(file_sizes): raise exceptions.StreamPushServeException( "Internal Error - HandleAddDbRequest:" " file_paths/file_sizes mismatch: %d/%d" % (len(file_paths), len(file_sizes))) db_timestamp = request.GetParameter(constants.DB_TIMESTAMP) db_size = request.GetParameter(constants.DB_SIZE) # Full ISO 8601 datetime format: '%Y-%m-%dT%H:%M:%S.%f-TZ'. # Check if database timestamp begins with a valid date format. if db_timestamp: try: datetime.datetime.strptime(db_timestamp[:10], serve_utils.DATE_ISO_FORMAT) except ValueError, e: logger.warning( "Database/Portable timestamp format is not valid. Error: %s", e) db_timestamp = None # Collecting database properties and packing into db_flags variable. db_use_google_basemap = request.GetParameter( constants.DB_USE_GOOGLE_BASEMAP) logger.debug("HandleAddDbRequest: db_use_google_basemap: %s", db_use_google_basemap) db_flags = 0 if db_use_google_basemap and int(db_use_google_basemap) == 1: db_flags = basic_types.DbFlags.USE_GOOGLE_BASEMAP (db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name) if serve_utils.IsFusionDb(db_type): client_host_name = request.GetClientHostName() if not client_host_name: raise exceptions.StreamPushServeException( "HandleAddDbRequest: missing Hostname.") elif serve_utils.IsPortable(db_type): # Note: The client host name is not used for portable globes, so we just # make it an empty string. client_host_name = "" # Get timestamp and size from portable info. db_info = basic_types.DbInfo() db_info.name = db_name db_info.type = db_type serve_utils.GlxDetails(db_info) db_timestamp = db_info.timestamp db_size = db_info.size else: raise exceptions.StreamPushServeException( "HandleAddDbRequest: Unsupported DB type %s.", db_type) # Check if database already exists. # The assumption is if it already exists, all the files and db-to-files # links have already been published to the database. db_id, curr_db_flags = self.QueryDbIdAndFlags(client_host_name, db_path) logger.debug("HandleAddDbRequest: db_id: %d, curr_db_flags: %d", db_id, curr_db_flags) if db_id > 0: # It's OK if the database already exists. Don't throw an exception. # Note: db_table.db_flags field was added in GEE 5.0.1, so for databases # pushed with GEE Fusion 5.0.0 and less we need to update db_flags in # postgres. if db_flags and (not curr_db_flags): # Update db_table.db_flags for existing database. query_string = ("UPDATE db_table SET db_flags = %s WHERE db_id = %s") self.DbModify(query_string, (db_flags, db_id)) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) return # We will now add the following entries into the db: # 1) the db_table entry for the database # 2) the files_table entries for each file path # 3) the db_files_table entries linking each file to this database # If anything fails in here, we want to remove the db_table entry (and any # db_files_table entries) so that publish will start from this point on # the next try. try: # Add the db entry. query_string = ( "INSERT INTO db_table (host_name, db_name, db_pretty_name," " db_timestamp, db_size, db_flags) " "VALUES(%s, %s, %s, %s, %s, %s)") self.DbModify(query_string, (client_host_name, db_name, db_pretty_name, db_timestamp, db_size, db_flags)) # Record the db_id now, if it succeeded, and the following inserts fail, # we can at least back some of the inserts out so we can try again later. db_id = self.QueryDbId(client_host_name, db_name) if db_id == 0: raise exceptions.StreamPushServeException( "HandleAddDbRequest: unable to add database: hostname:" " %s, database name: %s" % (client_host_name, db_name)) # Check which files already exist in the files_table. file_path_exists_flags = self._CheckFileExistence(client_host_name, file_paths) assert file_path_exists_flags # Add files entries. # Batch the SQL commands in groups of 1000 speeds things up noticeably. # fewer gets slightly worse, more is not noticeable. group_size = 1000 batcher = batch_sql_manager.BatchSqlManager( self.stream_db, group_size, logger) query_string = ("INSERT INTO files_table " "(host_name, file_path, file_size, file_status) " "VALUES(%s, %s, %s, 0)") for i in range(len(file_paths)): # Add only the newer files to avoid throwing SQLException. if not file_path_exists_flags[i]: batcher.AddModify( query_string, [client_host_name, file_paths[i], file_sizes[i]]) batcher.ExecuteRemaining() # Finally update the db_files_table by first querying the file_ids from # the files_table. file_ids = self._QueryFileIds(client_host_name, file_paths) assert file_ids query_string = ("INSERT INTO db_files_table (db_id, file_id) " "VALUES(%s, %s)") parameters = [(db_id, file_id) for file_id in file_ids] batcher.AddModifySet(query_string, parameters) batcher.ExecuteRemaining() batcher.Close() # Need to close to release SQL resources. http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) except Exception as e: # Record the internal exception to pass out via the final exception. error_message = ("HandleAddDbRequest:" " Error adding files to publisher database\n") + repr(e) # Clean up any remnants of this db_id, so that a subsequent attempt to # publish will start over at this step. # Note: this assumes that our connection is still valid. try: if db_id > 0: # Note: this will leave behind some entries in files_table, but this # will not hurt anything. query_string = "DELETE FROM db_files_table WHERE db_id = %s" self.DbModify(query_string, [db_id]) query_string = "DELETE FROM db_table WHERE db_id = %s" self.DbModify(query_string, [db_id]) except exceptions.Error: error_message += "\nAttempt to cleanup the database failed" raise exceptions.StreamPushServeException(error_message)
def HandleDeleteDbRequest(self, request, response): """Handles delete database requests. Args: request: request object. response: response object. Raises: OSError, psycopg2.Warning/Error, StreamPushServeException """ logger.debug("HandleDeleteDbRequest...") db_name = request.GetParameter(constants.DB_NAME) if not db_name: raise exceptions.StreamPushServeException( "HandleDeleteDbRequest: Missing database name.") (db_path, db_type) = serve_utils.IdentifyPublishedDb(db_name) if serve_utils.IsFusionDb(db_type): client_host_name = request.GetClientHostName() if not client_host_name: raise exceptions.StreamPushServeException( "HandleDeleteDbRequest: missing Hostname.") elif serve_utils.IsPortable(db_type): # Note: The client host name is not used for portable globes, so we just # make it an empty string. client_host_name = "" else: raise exceptions.StreamPushServeException( "HandleDeleteDbRequest: Unsupported DB type %s.", db_type) # Check if the database exists. db_id = self.QueryDbId(client_host_name, db_path) if db_id == 0: raise exceptions.StreamPushServeException( "HandleDeleteDbRequest: Could not find database: " "Fusion host: {0} Database name: {1}.".format( client_host_name, db_path)) # Check if the database is currently published. if self._QueryDbPublished(db_id): raise exceptions.StreamPushServeException( "HandleDeleteDbRequest: Database is currently published." " Please unpublish it first.") # Delete the entries in db_table. query_string = "DELETE FROM db_table WHERE db_id = %s" self.DbModify(query_string, (db_id,)) if serve_utils.IsFusionDb(db_type): # Delete the entries in db_files_table. The entries in the # files_table and the actual files are untouched. Those will be garbage # collected at the request of the user. query_string = "DELETE FROM db_files_table WHERE db_id = %s" self.DbModify(query_string, [db_id]) elif serve_utils.IsPortable(db_type): query_string = ("SELECT file_id FROM db_files_table WHERE db_id = %s") rs_db_files = self.DbQuery(query_string, (db_id,)) if rs_db_files: assert len(rs_db_files) == 1 file_id = rs_db_files[0] # Delete the entries in db_files_table. delete_db_files_table_cmd = ( "DELETE FROM db_files_table WHERE db_id = %s") self.DbModify(delete_db_files_table_cmd, (db_id,)) # Get file_path from files_table and delete file. query_string = ("SELECT file_path FROM files_table" " WHERE file_id = %s") rs_files = self.DbQuery(query_string, (file_id,)) if rs_files: assert len(rs_files) == 1 assert db_path == rs_files[0] if os.path.exists(rs_files[0]): os.unlink(rs_files[0]) # Delete the file entry from files_table. delete_files_table_cmd = "DELETE FROM files_table WHERE file_id = %s" self.DbModify(delete_files_table_cmd, (file_id,)) else: raise exceptions.StreamPushServeException( "Unsupported DB type %s.", db_type) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS)
def HandleQueryRequest(self, request, response): """Handles query requests. Args: request: request object. response: response object. Raises: StreamPushServeException. """ query_cmd = request.GetParameter(constants.QUERY_CMD) if not query_cmd: raise exceptions.StreamPushServeException( "Internal Error - Missing Query Command.") # List all DBs registered on server. if query_cmd == constants.QUERY_CMD_LIST_DBS: query_string = "SELECT host_name, db_name, db_pretty_name FROM db_table" results = self.DbQuery(query_string) for line in results: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_HOST_NAME, line[0]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DB_NAME, line[1]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DB_PRETTY_NAME, line[2]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # List all published DBs. elif query_cmd == constants.QUERY_CMD_PUBLISHED_DBS: query_string = """ SELECT target_table.target_path, virtual_host_table.virtual_host_name, db_table.db_name, db_table.db_pretty_name, db_table.host_name FROM db_table, virtual_host_table, target_db_table, target_table WHERE db_table.db_id = target_db_table.db_id AND target_table.target_id = target_db_table.target_id AND virtual_host_table.virtual_host_id = target_db_table.virtual_host_id """ results = self.DbQuery(query_string) for line in results: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_TARGET_PATH, line[0]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_VS_NAME, line[1]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DB_NAME, line[2]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_DB_PRETTY_NAME, line[3]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_HOST_NAME, line[4]) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get DB details. elif query_cmd == constants.QUERY_CMD_DB_DETAILS: db_name = request.GetParameter(constants.DB_NAME) if not db_name: raise exceptions.StreamPushServeException( "HandleQueryRequest: missing db name") client_host_name = request.GetClientHostName() if not client_host_name: raise exceptions.StreamPushServeException( "HandleQueryRequest: missing Hostname.") query_string = ( "SELECT file_path FROM files_table WHERE file_id IN (" "SELECT file_id FROM db_files_table WHERE db_id IN (" "SELECT db_id FROM db_table WHERE " "host_name = %s AND db_name = %s))") results = self.DbQuery(query_string, (client_host_name, db_name)) for line in results: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_FILE_NAME, line) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get server prefix - path to published_dbs directory. elif query_cmd == constants.QUERY_CMD_SERVER_PREFIX: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_SERVER_PREFIX, self.server_prefix) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get client hostname. elif query_cmd == constants.QUERY_CMD_HOST_ROOT: client_host_name = request.GetClientHostName() if not client_host_name: raise exceptions.StreamPushServeException( "HandleQueryRequest: missing Hostname.") http_io.ResponseWriter.AddBodyElement( response, constants.HDR_HOST_ROOT, client_host_name) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get server hostname. elif query_cmd == constants.QUERY_CMD_SERVER_HOST: host_name = socket.gethostname() full_host_name = socket.getfqdn() http_io.ResponseWriter.AddBodyElement( response, constants.HDR_SERVER_HOST, host_name) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_SERVER_HOST_FULL, full_host_name) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) # Get allow symbolic link parameter. elif query_cmd == constants.QUERY_CMD_ALLOW_SYM_LINKS: http_io.ResponseWriter.AddBodyElement( response, constants.HDR_ALLOW_SYM_LINKS, self._allow_symlinks) http_io.ResponseWriter.AddBodyElement( response, constants.HDR_STATUS_CODE, constants.STATUS_SUCCESS) else: raise exceptions.StreamPushServeException( "Internal Error - Invalid Query Command: %s." % query_cmd)