def main(): """Entry point for this utility. Usage:: $ python createApiUser.py """ try: ## Setup requested log handlers globalSettings = utils.loadSettings( os.path.join(env.configPath, "globalSettings.json")) logFiles = utils.setupLogFile( "ApiApplication", env, globalSettings['fileContainingServiceLogSettings'], directoryName='service') logObserver = utils.setupObservers( logFiles, "ApiApplication", env, globalSettings['fileContainingServiceLogSettings']) logger = twisted.logger.Logger(observer=logObserver, namespace="ApiApplication") logger.info('Starting createApiUser') ## Connect to database dbClient = DatabaseClient(logger) if dbClient is None: raise SystemError( 'Failed to connect to database; unable to initialize tables.') ## Get list of valid users users = {} getUsers(dbClient, logger, users) ## Create and insert a new credential createUserEntry(dbClient, logger, users) ## Cleanup dbClient.session.remove() dbClient.close() logger.info('Exiting configureDatabase utility.') except: stacktrace = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) ## The basic print is here for a console message in case we weren't ## able to use the logging mechanism before encountering the failure. print('Failure in createApiUser: {}'.format(stacktrace)) try: logger.debug('Failure in createApiUser: {}'.format(stacktrace)) except: pass ## end main return
def main(): """Entry point for the database configuration utility. Usage:: $ python rebuildDataSchema.py """ try: ## Setup requested log handlers logEntity = 'Database' logger = utils.setupLogger(logEntity, env, 'logSettingsCore.json') logger.info('Starting rebuildDataSchema utility.') ## Attempt connection dbClient = DatabaseClient(logger) if dbClient is None: raise SystemError( 'Failed to connect to database; unable to rebuild data schema.' ) print('\nDatabase connection successful.') logger.debug('Database connection successful') ## Start the work createTables(dbClient, logger) ## Close the connection dbClient.close() logger.info('Exiting rebuildDataSchema utility.') except: stacktrace = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) ## The basic print is here for a console message in case we weren't ## able to use the logging mechanism before encountering the failure. print('Failure in rebuildDataSchema.main: {}'.format(stacktrace)) try: logger.debug( 'Failure in rebuildDataSchema.main: {}'.format(stacktrace)) except: pass ## end main return
def getDbConnection(logger): ## Attempt connection dbClient = DatabaseClient(logger) if dbClient is None: raise SystemError('Failed to connect to database.') ## end getDbConnection return dbClient
def main(): """Entry point for this utility. Usage:: $ python createProtocolEntry.py """ try: ## Setup requested log handlers logEntity = 'Protocols' logger = utils.setupLogger(logEntity, env, 'logSettingsCore.json') logger.info('Starting manageProtocols.') ## Connect to database dbClient = DatabaseClient(logger) if dbClient is None: raise SystemError('Failed to connect to database; unable to initialize tables.') ## Get list of valid realms realms = [] getRealms(dbClient, logger, realms) ## Create and insert a new credential createProtocolEntry(dbClient, logger, realms) ## Cleanup dbClient.session.remove() dbClient.close() logger.info('Exiting configureDatabase utility.') except: stacktrace = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) ## The basic print is here for a console message in case we weren't ## able to use the logging mechanism before encountering the failure. print('Failure in configureDatabase: {}'.format(stacktrace)) try: logger.debug('Failure in configureDatabase: {}'.format(stacktrace)) except: pass ## end main return
def getSharedDbPool(self): """Create a global context for sharing a DB connection pool. When using a scoped_session in sqlalchemy, the code checks if there was a previous thread-local session created. If one exists then sqlalchemy will simply reuse it. This enables a shared pool for API connections. """ dbClient = DatabaseClient(self.logger, globalSettings=self.globalSettings, env=env, poolSize=20, maxOverflow=7, poolRecycle=1800) if dbClient is None: self.logger.error('Failed to connect to shared database pool.') raise EnvironmentError( 'Failed to connect to shared database pool.') dbClient.session.close() dbClient.session.remove() dbClient.close() self.logger.info('Created the SqlAlchemy connection pool.') ## end getSharedDbPool return
def getDbSession(self): """Get instance for database client from defined configuration.""" ## If the database isn't up when the client is starting... wait on it. self.logger.debug('Attempting to connect to database') while self.dbClient is None and not self.shutdownEvent.is_set( ) and not self.canceledEvent.is_set(): try: ## Hard coding the connection pool settings for now; may want to ## pull into a localSetting if they need to be independently set self.dbClient = DatabaseClient( self.logger, globalSettings=self.globalSettings, env=env, poolSize=1, maxOverflow=2, poolRecycle=900) if self.dbClient is None: self.canceledEvent.set() raise EnvironmentError('Failed to connect to database') self.logger.debug('Database connection successful') except exc.OperationalError: self.logger.debug( 'DB is not available; waiting {waitCycle!r} seconds before next retry.', waitCycle=self.globalSettings[ 'waitSecondsBeforeRetryingDatabaseConnection']) self.logToKafka( 'DB is not available; waiting {waitCycle!r} seconds before next retry.' .format(self.globalSettings[ 'waitSecondsBeforeRetryingDatabaseConnection'])) time.sleep( int(self.globalSettings[ 'waitSecondsBeforeRetryingDatabaseConnection'])) except: exception = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) self.logger.error('Exception: {exception!r}', exception=exception) self.canceledEvent.set() self.logToKafka(sys.exc_info()[1]) break ## end getDbSession return
def getDbSession(self, maxAttempts=24, waitSeconds=5): """Get database connection; wait number of times or abort.""" ## If the database isn't up when the client is starting... wait on it; ## currently set to retry every 5 sec for 2 minutes, before failing. self.logger.debug('Attempting to connect to database') errorCount = 0 while (self.dbClient is None and not self.shutdownEvent.is_set() and not self.canceledEvent.is_set() and errorCount < maxAttempts): try: ## Hard coding the connection pool settings for now; may want to ## pull into a setting if they need to be set per service self.dbClient = DatabaseClient( self.logger, globalSettings=self.globalSettings, env=env, poolSize=2, maxOverflow=1, poolRecycle=900) if self.dbClient is None: self.canceledEvent.set() raise EnvironmentError('Failed to connect to database') self.logger.debug('Database connection successful') except exc.OperationalError: self.logger.debug( 'DB is not available; waiting {waitCycle!r} seconds before next retry.', waitCycle=waitSeconds) time.sleep(int(waitSeconds)) except: exception = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) self.logger.error('Exception: {exception!r}', exception=exception) self.canceledEvent.set() break errorCount += 1 return
## trying to rotate after the size limit with I/O errors on closing the file. So ## to handle multi-process/thread access, we use concurrent-log-handler. #logger = utils.setupLogger('ApiApplication', env, 'logSettingsServices.json', directoryName='service') logger = logging.getLogger('ApiApplication') logger.info('Starting apiResourceRoot') ## Create a global context for sharing the DB connection pool. When using a ## scoped_session in sqlalchemy, the code checks if there is was a previous ## thread-local session created. When one already exists, sqlalchemy will simply ## reuse it. And in order to get the thread-local session to URL functions, we ## use a middleware class (SQLAlchemySessionMiddleware below). This allows us to ## share database resources across API connections. from database.connectionPool import DatabaseClient dbClient = DatabaseClient(logger, globalSettings=globalSettings, env=env, poolSize=20, maxOverflow=7, poolRecycle=1800) if dbClient is None: logger.error('Failed to connect to the shared database connection pool.') raise EnvironmentError( 'Failed to connect to the shared database connection pool.') dbClient.session.close() dbClient.session.remove() dbClient.close() logger.info('Acquired a handle to the shared database connection pool.') ############################################################################# ## Falcon middleware section #############################################################################