def __init__(self, logger): self._logger = logger self._staticPathInLocation = "ezbappstatic" self._scHash = dict( ) # stores {urlPrefix: hash} of static contents directory self._sfsHash = dict( ) # stores {urlPrefix: hash} of static contents in Static File Store self._curDir = None self._buildDir = None accConfig = AccumuloConfiguration(gConfig.ezproperties) self._sFileStore = EzRPStaticStore( host=accConfig.getProxyHost(), port=accConfig.getProxyPort(), user=accConfig.getUsername(), password=accConfig.getPassword(), chunk_size=int(gConfig.static_content_chunk_size), logger=self._logger)
def __init__(self, logger): self._logger = logger self._staticPathInLocation = "ezbappstatic" self._scHash = dict() # stores {urlPrefix: hash} of static contents directory self._sfsHash = dict() # stores {urlPrefix: hash} of static contents in Static File Store self._curDir = None self._buildDir = None accConfig = AccumuloConfiguration(gConfig.ezproperties) self._sFileStore = EzRPStaticStore(host=accConfig.getProxyHost(), port=accConfig.getProxyPort(), user=accConfig.getUsername(), password=accConfig.getPassword(), chunk_size=int(gConfig.static_content_chunk_size), logger=self._logger)
class StaticFileHandler(object): ''' Class to manage Static File serving using Static File Store ''' def __init__(self, logger): self._logger = logger self._staticPathInLocation = "ezbappstatic" self._scHash = dict( ) # stores {urlPrefix: hash} of static contents directory self._sfsHash = dict( ) # stores {urlPrefix: hash} of static contents in Static File Store self._curDir = None self._buildDir = None accConfig = AccumuloConfiguration(gConfig.ezproperties) self._sFileStore = EzRPStaticStore( host=accConfig.getProxyHost(), port=accConfig.getProxyPort(), user=accConfig.getUsername(), password=accConfig.getPassword(), chunk_size=int(gConfig.static_content_chunk_size), logger=self._logger) def _removeFromStaticFileStore(self, urlPrefix): hash_value = self._sfsHash.get(urlPrefix) if hash_value: self._logger.info( 'Removing Static File fromStaticFileStore for {0} with hash {1}' .format(urlPrefix, hash_value)) self._sFileStore.deleteFile(urlPrefix) else: raise Exception('Content not found') def _copyFromStaticFileStore(self, urlPrefix, contentServiceType): ''' Gets the static file from Static File Store if available and put it in static contents directory ''' status = False hash_value = self._sfsHash.get(urlPrefix) if hash_value: bdata = self._sFileStore.getFile(urlPrefix) if bdata is not None and archive.is_data_archive(bdata): self._logger.debug( 'Getting Static File from Static File Store for {0} with hash {1}' .format(urlPrefix, hash_value)) status = self._extractStaticFiles(bdata, hash_value, urlPrefix, contentServiceType) else: # try one more time self._sFileStore.reConnection() bdata = self._sFileStore.getFile(urlPrefix) if bdata is not None and archive.is_data_archive(bdata): self._logger.debug( 'Getting Static File from Static File Store for {0} with hash {1} after retrying' .format(urlPrefix, hash_value)) status = self._extractStaticFiles(bdata, hash_value, urlPrefix, contentServiceType) else: # Delete Static File Store Content; files may be corrupted while uploading to Static File Store self._sFileStore.deleteFile(urlPrefix) return status def _extractStaticFiles(self, bdata, hash_value, urlPrefix, contentServiceType): ''' Extract static files into ezappStaticdir ''' try: tfile = utils.getTempfile() with open(tfile, "w+b") as f: f.write(bdata) url_hash = str(urlPrefix + "_" + hash_value) if contentServiceType == ContentServiceType.HYBRID: filelist = [ archive.Archive(path_to_extract=os.path.join( self._buildDir, url_hash, urlPrefix, self._staticPathInLocation), archive_file=tfile) ] else: filelist = [ archive.Archive(path_to_extract=os.path.join( self._buildDir, url_hash, urlPrefix), archive_file=tfile) ] filesToExtract = archive.openArchive(filelist) self._logger.debug( 'Extracting archive_file "{0}" into path "{1}"'.format( filelist[0].archive_file, filelist[0].path_to_extract)) archive.extractArchive(filesToExtract) utils.remove_file_if_exists(tfile) return True except Exception as e: utils.remove_file_if_exists(tfile) self._logger.exception( 'Exception while Extracting: urlPrefix {0}}'.format(urlPrefix)) return False def _getCurrentStaticDirEntry(self): ''' Generator gets the urlPrefix and hash of all the static contents the Nginx is currently serving The directory structure is in this format: /opt/ezfrontend/ezbappstatic/ <current_static_path> / /* FOR HYBRID */ |-- urlPrefix_hash/app1/ezbappstatic/-- Static Contents | /* FOR STATIC_ONLY */ |-- urlPrefix_hash/app1/...static Contents ''' dirlist = utils.rootDirs( self._curDir ) # Get all the directories as seen from current_static_path for url_hash in dirlist: urlPrefix, hash_value = url_hash.split('_') self._logger.debug( "_getCurrentStaticDirEntry urlPrefix={0} hash={1}".format( urlPrefix, hash_value)) yield (urlPrefix, hash_value) else: yield (None, None) def _createBuildDir(self): ''' Sets a build path based on current static path ''' self._curDir = os.readlink(gConfig.static_contents) self._buildDir = random.choice( [x for x in gConfig.static_contents_dirs if x != self._curDir]) utils.ensurePathExists(self._buildDir) self._logger.info("createBuildDir current dir {0} builddir {1}".format( self._curDir, self._buildDir)) def _copyFromLocalDir(self, urlPrefix): ''' Copy the static contents of urlPrefix of _previous_static_path into _current_static_path ''' try: hash_value = self._scHash.get(urlPrefix) if not hash_value: raise ValueError( 'Could not get hash for urlPrefix "{0}"'.format(urlPrefix)) url_hash = str(urlPrefix + "_" + hash_value) previous_path = os.path.join(self._curDir, url_hash) current_path = os.path.join(self._buildDir, url_hash) utils.copyPath(str(previous_path), str(current_path)) return True except Exception as e: self._logger.error( 'Error while copying static contents from "{0}" to "{1}" : {2}' .format(previous_path, current_path, str(e))) return False def staticLocationForNginx(self, urlPrefix): ''' Returns the path of static content to urlPrefix Used while generating Nginx Configuration ''' hash_str = self._sfsHash.get(urlPrefix) if hash_str: return os.path.join(self._buildDir, str(urlPrefix + "_" + hash_str)) else: self._logger.error( 'Error while getting the root Path of StaticContent for Nginx; urlPrefix "{0}": The hash content is None' .format(urlPrefix)) return None def _putInStaticFileStore(self, urlPrefix, data): hash_str = self._sfsHash.get(urlPrefix) hash_data = utils.hash_md5(data) if hash_str == hash_data: raise RuntimeError( 'Static content already exists for {0}'.format(urlPrefix)) self._sFileStore.putFile(urlPrefix, hash_data, data) self._sfsHash[urlPrefix] = hash_data def _addInStaticFileStore(self, urlPrefix, data): ''' If file is in archive format, and not already in Static File Store Update Static File Store and trigger ZookeeperWatch Node. ''' static_content_max_size = int(gConfig.static_content_max_size) if 0 < len(data) > static_content_max_size: raise ValueError( 'Data for {0} is empty or exceeds the limit {1}MB'.format( urlPrefix, static_content_max_size / 1048576)) if not archive.is_data_archive(data): raise StaticContentException(message='Not an Archive', urls=[urlPrefix]) self._putInStaticFileStore(urlPrefix, data) def addStaticContent(self, contents): ''' Saves contents received through Thrift in Static File Store ''' try: for staticContent in contents: self._logger.info('addStaticContent urlPrefix "{0}"'.format( staticContent.userFacingUrlPrefix)) self._addInStaticFileStore(staticContent.userFacingUrlPrefix, staticContent.content) return True except Exception as e: self._logger.error( 'Error while adding static content for urlPrefix "{0}"; Reason: {1}' .format(staticContent.userFacingUrlPrefix, str(e))) raise StaticContentException( message=str(e), urls=[staticContent.userFacingUrlPrefix]) def removeStaticContent(self, contents): ''' Deletes static contents of urlPrefix from Static File Store ''' try: for staticContent in contents: self._logger.info('removeStaticContent urlPrefix "{0}"'.format( staticContent.userFacingUrlPrefix)) self._removeFromStaticFileStore( staticContent.userFacingUrlPrefix) del self._sfsHash[staticContent.userFacingUrlPrefix] return True except Exception as e: emsg = 'Error while removing static content for urlPrefix "{0}"; Reason: {1}'.format( staticContent.userFacingUrlPrefix, str(e)) self._logger.error('{0}'.format(emsg)) raise StaticContentException( message=emsg, url=[staticContent.userFacingUrlPrefix]) def isStaticContentPresent(self, urlPrefix): ''' Gets the hash value for urlPrefix from Static File Store ''' if self._sfsHash.get(urlPrefix): return True else: return False def updateStaticDirLink(self): ''' Deletes current static path being served by Nginx. This is called after generating nginx configuration file and just before reconfiguration. ''' self._logger.info( 'deleted StaticContents Dir "{0}" created link for Dir "{1}"'. format(self._curDir, self._buildDir)) if utils.ifPathExists(self._curDir): utils.deletePath(self._curDir) subprocess.call( ['ln', '-sTf', self._buildDir, gConfig.static_contents]) def updateStaticContentsDict(self): ''' Update the: Dictionary which holds the current static contents in Static File Store and the Dictionary which holds the current static contents in ezappStaticdir A call to this signals that Nginx is configuring, switch the static path for configuring ''' self._createBuildDir() if not self._sfsHash: # Update dict while startup self._sfsHash = { urlPrefix: hash_value for urlPrefix, hash_value in self._sFileStore.getAttributes() } self._scHash = { urlPrefix: hash_value for urlPrefix, hash_value in self._getCurrentStaticDirEntry() } def updateStaticDir(self, urlPrefix, contentServiceType): ''' This will be called form ezRPConfigNginx when static content is needed for urlPrefix ''' try: self._logger.info( 'Updating Nginx Static dir for urlPrefix "{0}"'.format( urlPrefix)) # static contents hash of local and Static File Store scHash = self._scHash.get(urlPrefix) sfsHash = self._sfsHash.get(urlPrefix) # **** Cases **** # No content available for this urlPrefix in Static File Store # Then don't update the contents if not sfsHash: self._logger.debug( 'No Static File Store content for urlPrefix "{0}" found, static contents not updated' .format(urlPrefix)) return False # Both Static File Store and Local Static contents are available if sfsHash != scHash: self._logger.debug( 'Found latest content for urlPrefix "{0}" in Static File Store. Updating' .format(urlPrefix)) return self._copyFromStaticFileStore(urlPrefix, contentServiceType) else: # Just copy it from working to build directory self._logger.debug( 'Copying the static contents of urlPrefix "{0}" from the local ezbappstatic dir' .format(urlPrefix)) return self._copyFromLocalDir(urlPrefix) except Exception as e: ''' Any Error, we should not update the link in the configuration ''' self._logger.error( 'Error occured while updating static dir: {0}'.format(str(e))) return False
class StaticFileHandler(object): ''' Class to manage Static File serving using Static File Store ''' def __init__(self, logger): self._logger = logger self._staticPathInLocation = "ezbappstatic" self._scHash = dict() # stores {urlPrefix: hash} of static contents directory self._sfsHash = dict() # stores {urlPrefix: hash} of static contents in Static File Store self._curDir = None self._buildDir = None accConfig = AccumuloConfiguration(gConfig.ezproperties) self._sFileStore = EzRPStaticStore(host=accConfig.getProxyHost(), port=accConfig.getProxyPort(), user=accConfig.getUsername(), password=accConfig.getPassword(), chunk_size=int(gConfig.static_content_chunk_size), logger=self._logger) def _removeFromStaticFileStore(self, urlPrefix): hash_value = self._sfsHash.get(urlPrefix) if hash_value: self._logger.info('Removing Static File fromStaticFileStore for {0} with hash {1}'.format(urlPrefix, hash_value)) self._sFileStore.deleteFile(urlPrefix) else: raise Exception('Content not found') def _copyFromStaticFileStore(self, urlPrefix, contentServiceType): ''' Gets the static file from Static File Store if available and put it in static contents directory ''' status = False hash_value = self._sfsHash.get(urlPrefix) if hash_value: bdata = self._sFileStore.getFile(urlPrefix) if bdata is not None and archive.is_data_archive(bdata): self._logger.debug('Getting Static File from Static File Store for {0} with hash {1}'.format(urlPrefix, hash_value)) status = self._extractStaticFiles(bdata, hash_value, urlPrefix, contentServiceType) else: # try one more time self._sFileStore.reConnection() bdata = self._sFileStore.getFile(urlPrefix) if bdata is not None and archive.is_data_archive(bdata): self._logger.debug('Getting Static File from Static File Store for {0} with hash {1} after retrying'.format(urlPrefix, hash_value)) status = self._extractStaticFiles(bdata, hash_value, urlPrefix, contentServiceType) else: # Delete Static File Store Content; files may be corrupted while uploading to Static File Store self._sFileStore.deleteFile(urlPrefix) return status def _extractStaticFiles(self, bdata, hash_value, urlPrefix, contentServiceType): ''' Extract static files into ezappStaticdir ''' try: tfile = utils.getTempfile() with open(tfile, "w+b") as f: f.write(bdata) url_hash = str(urlPrefix + "_" + hash_value) if contentServiceType == ContentServiceType.HYBRID: filelist = [archive.Archive(path_to_extract=os.path.join(self._buildDir, url_hash, urlPrefix, self._staticPathInLocation), archive_file=tfile)] else: filelist = [archive.Archive(path_to_extract=os.path.join(self._buildDir, url_hash, urlPrefix), archive_file=tfile)] filesToExtract = archive.openArchive(filelist) self._logger.debug('Extracting archive_file "{0}" into path "{1}"'.format(filelist[0].archive_file, filelist[0].path_to_extract)) archive.extractArchive(filesToExtract) utils.remove_file_if_exists(tfile) return True except Exception as e: utils.remove_file_if_exists(tfile) self._logger.exception('Exception while Extracting: urlPrefix {0}}'.format(urlPrefix)) return False def _getCurrentStaticDirEntry(self): ''' Generator gets the urlPrefix and hash of all the static contents the Nginx is currently serving The directory structure is in this format: /opt/ezfrontend/ezbappstatic/ <current_static_path> / /* FOR HYBRID */ |-- urlPrefix_hash/app1/ezbappstatic/-- Static Contents | /* FOR STATIC_ONLY */ |-- urlPrefix_hash/app1/...static Contents ''' dirlist = utils.rootDirs(self._curDir) # Get all the directories as seen from current_static_path for url_hash in dirlist: urlPrefix, hash_value = url_hash.split('_') self._logger.debug("_getCurrentStaticDirEntry urlPrefix={0} hash={1}".format(urlPrefix, hash_value)) yield (urlPrefix, hash_value) else: yield (None, None) def _createBuildDir(self): ''' Sets a build path based on current static path ''' self._curDir = os.readlink(gConfig.static_contents) self._buildDir = random.choice([x for x in gConfig.static_contents_dirs if x != self._curDir]) utils.ensurePathExists(self._buildDir) self._logger.info("createBuildDir current dir {0} builddir {1}".format(self._curDir, self._buildDir)) def _copyFromLocalDir(self, urlPrefix): ''' Copy the static contents of urlPrefix of _previous_static_path into _current_static_path ''' try: hash_value = self._scHash.get(urlPrefix) if not hash_value: raise ValueError('Could not get hash for urlPrefix "{0}"'.format(urlPrefix)) url_hash = str(urlPrefix + "_" + hash_value) previous_path = os.path.join(self._curDir, url_hash) current_path = os.path.join(self._buildDir, url_hash) utils.copyPath(str(previous_path), str(current_path)) return True except Exception as e: self._logger.error('Error while copying static contents from "{0}" to "{1}" : {2}'.format(previous_path, current_path, str(e))) return False def staticLocationForNginx(self, urlPrefix): ''' Returns the path of static content to urlPrefix Used while generating Nginx Configuration ''' hash_str = self._sfsHash.get(urlPrefix) if hash_str: return os.path.join(self._buildDir, str(urlPrefix + "_" + hash_str)) else: self._logger.error('Error while getting the root Path of StaticContent for Nginx; urlPrefix "{0}": The hash content is None'.format(urlPrefix)) return None def _putInStaticFileStore(self, urlPrefix, data): hash_str = self._sfsHash.get(urlPrefix) hash_data = utils.hash_md5(data) if hash_str == hash_data: raise RuntimeError('Static content already exists for {0}'.format(urlPrefix)) self._sFileStore.putFile(urlPrefix, hash_data, data) self._sfsHash[urlPrefix]=hash_data def _addInStaticFileStore(self, urlPrefix, data): ''' If file is in archive format, and not already in Static File Store Update Static File Store and trigger ZookeeperWatch Node. ''' static_content_max_size = int(gConfig.static_content_max_size) if 0 < len(data) > static_content_max_size: raise ValueError('Data for {0} is empty or exceeds the limit {1}MB'.format(urlPrefix, static_content_max_size/1048576)) if not archive.is_data_archive(data): raise StaticContentException(message='Not an Archive', urls=[urlPrefix]) self._putInStaticFileStore(urlPrefix, data) def addStaticContent(self, contents): ''' Saves contents received through Thrift in Static File Store ''' try: for staticContent in contents: self._logger.info('addStaticContent urlPrefix "{0}"'.format(staticContent.userFacingUrlPrefix)) self._addInStaticFileStore(staticContent.userFacingUrlPrefix, staticContent.content) return True except Exception as e: self._logger.error('Error while adding static content for urlPrefix "{0}"; Reason: {1}'.format(staticContent.userFacingUrlPrefix, str(e))) raise StaticContentException(message=str(e), urls=[staticContent.userFacingUrlPrefix]) def removeStaticContent(self, contents): ''' Deletes static contents of urlPrefix from Static File Store ''' try: for staticContent in contents: self._logger.info('removeStaticContent urlPrefix "{0}"'.format(staticContent.userFacingUrlPrefix)) self._removeFromStaticFileStore(staticContent.userFacingUrlPrefix) del self._sfsHash[staticContent.userFacingUrlPrefix] return True except Exception as e: emsg = 'Error while removing static content for urlPrefix "{0}"; Reason: {1}'.format(staticContent.userFacingUrlPrefix, str(e)) self._logger.error('{0}'.format(emsg)) raise StaticContentException(message=emsg, url=[staticContent.userFacingUrlPrefix]) def isStaticContentPresent(self, urlPrefix): ''' Gets the hash value for urlPrefix from Static File Store ''' if self._sfsHash.get(urlPrefix): return True else: return False def updateStaticDirLink(self): ''' Deletes current static path being served by Nginx. This is called after generating nginx configuration file and just before reconfiguration. ''' self._logger.info('deleted StaticContents Dir "{0}" created link for Dir "{1}"'.format(self._curDir, self._buildDir)) if utils.ifPathExists(self._curDir): utils.deletePath(self._curDir) subprocess.call(['ln', '-sTf', self._buildDir, gConfig.static_contents]) def updateStaticContentsDict(self): ''' Update the: Dictionary which holds the current static contents in Static File Store and the Dictionary which holds the current static contents in ezappStaticdir A call to this signals that Nginx is configuring, switch the static path for configuring ''' self._createBuildDir() if not self._sfsHash: # Update dict while startup self._sfsHash = {urlPrefix: hash_value for urlPrefix, hash_value in self._sFileStore.getAttributes()} self._scHash = {urlPrefix: hash_value for urlPrefix, hash_value in self._getCurrentStaticDirEntry()} def updateStaticDir(self, urlPrefix, contentServiceType): ''' This will be called form ezRPConfigNginx when static content is needed for urlPrefix ''' try: self._logger.info('Updating Nginx Static dir for urlPrefix "{0}"'.format(urlPrefix)) # static contents hash of local and Static File Store scHash = self._scHash.get(urlPrefix) sfsHash = self._sfsHash.get(urlPrefix) # **** Cases **** # No content available for this urlPrefix in Static File Store # Then don't update the contents if not sfsHash: self._logger.debug('No Static File Store content for urlPrefix "{0}" found, static contents not updated'.format(urlPrefix)) return False # Both Static File Store and Local Static contents are available if sfsHash != scHash: self._logger.debug('Found latest content for urlPrefix "{0}" in Static File Store. Updating'.format(urlPrefix)) return self._copyFromStaticFileStore(urlPrefix, contentServiceType) else: # Just copy it from working to build directory self._logger.debug('Copying the static contents of urlPrefix "{0}" from the local ezbappstatic dir'.format(urlPrefix)) return self._copyFromLocalDir(urlPrefix) except Exception as e: ''' Any Error, we should not update the link in the configuration ''' self._logger.error('Error occured while updating static dir: {0}'.format(str(e))) return False