def __init__(self): settings = dict( cookie_secret="2c-*JKf74*41pfgj8.J|0G\Iuv@H:!,x]%V", template_path=os.path.join(os.path.dirname(__file__), "templates"), static_path=os.path.join(os.path.dirname(__file__), "static"), xsrf_cookies=False, autoescape=None ) mongo_uri = os.environ.get('MONGOLAB_URI', False) if mongo_uri: regex = re.compile("^(?P<scheme>.*):\/\/(?P<user>.*):(?P<password>.*)@(?P<host>.*):(?P<port>\d*)\/(?P<database>.*)$") creds = regex.search(mongo_uri).groupdict() mongo = txmongo.lazyMongoConnectionPool(host=creds['host'], port=int(creds['port'])) db = mongo[creds['database']] db.authenticate(creds['user'], creds['password']) else: log.msg("Running locally, connectiong to local mongodb without authentication.") mongo = txmongo.lazyMongoConnectionPool() db = mongo['valldb'] handlers = [ (r"/(.*)/scripts.js", JSHandler), (r"/(.*)", VideowallHandler, dict(db=db)), (r"/static/(.*)", cyclone.web.StaticFileHandler, dict(path=settings['static_path'])), ] cyclone.web.Application.__init__(self, handlers, **settings)
def test_lazyMongoConnectionPool(self): # lazyMongoConnection returns MongoAPI rapi = txmongo.lazyMongoConnectionPool(mongo_host, mongo_port, pool_size=2) self.assertEqual(isinstance(rapi, txmongo.MongoAPI), True) yield rapi._connected disconnected = yield rapi.disconnect() self.assertEqual(disconnected, True)
def _init_search(cls): """ Initializes everything needed for search. """ config_path = cls.search_config_path if not os.path.exists(config_path): raise OSError(errno.ENOENT, "Config %r does not exist." % config_path, config_path) config_dir = os.path.dirname(config_path) # Read config. with open(config_path, 'rb') as fh: config = json.load(fh) cls.search_config = config # Connect to mongo. host = config['mongo']['host'] port = config['mongo'].get('port', None) or 27017 thread_pool = reactor.getThreadPool() pool_size = int(math.ceil((thread_pool.min + thread_pool.max) / 2)) cls.search_mongo = txmongo.lazyMongoConnectionPool(host=host, port=port, pool_size=pool_size) cls.search_order_db = cls.search_mongo[config['mongo']['order_dbname']] cls.search_order_tb = cls.search_order_db[config['mongo']['order_tbname']] # Initialize PyLucene. lucene.initVM() # Open index. index_path = os.path.abspath(os.path.join(config_dir, config['lucene']['index_path'])) if not os.path.exists(index_path): raise OSError(errno.ENOENT, "Index %r does not exist." % index_path, index_path) elif not os.path.isdir(index_path): raise OSError(errno.ENOTDIR, "Index %r is not a directory." % index_path, index_path) index_dir = lucene.NIOFSDirectory(lucene.File(index_path)) #index_dir = lucene.SimpleFSDirectory(lucene.File(index_path)) # windows cls.search_searcher = lucene.IndexSearcher(index_dir)
def setup(): _db = txmongo.lazyMongoConnectionPool(MONGO_SERVER, 27017) collection = _db.houseprices.houses root = resource.Resource() # Can not do a 'distinct' query with async mongo - so use a sync connection for this sync_connection = pymongo.Connection(MONGO_SERVER, 27017, max_pool_size=10) sync_db = sync_connection ["houseprices"] dates = sync_db.houses.distinct( 'dateadded') legal_postcodes_results = sync_db.houses.distinct( 'postcode_part') postcodes = [] for p in legal_postcodes_results: postcodes.append(p) postcodes.sort() sync_connection.close() #root = ElementResource(db) #root.putChild("/favicon.ico", FaviconHandler) root.putChild("/", PriceElementResource(dates, postcodes, collection)) root.putChild("price", PriceElementResource(dates, postcodes, collection)) root.putChild("postcode", PostcodeElementResource(dates, postcodes, collection)) root.putChild("single", SingleElementResource(dates, collection)) root.putChild("hello", HelloHandler()) factory = Site(root) reactor.listenTCP(8881, factory) reactor.run()
def makeService(self, options): # MongoDB mongo.connection.set(txmongo.lazyMongoConnectionPool( **settings.mongodb['connection'])) module_service = super(ArchiverMaker, self).makeService(options) transfer_portal = portal.Portal(self.ftp_realm, [ checkers.HomeDirectoryExistence(settings.uploads_root, settings.downloads_root) ]) ## #context = ssl.DefaultOpenSSLContextFactory( # settings.server_key, # settings.server_crt #) # #ftp_service = internet.SSLServer(settings.ftp_server_port, # ftp.FTPFactory(transfer_portal), # context, # interface=settings.ftp_server_ip) ## conv_service = converter.ConversionService(settings.conv_server_port, interface=settings.conv_server_ip) conv_service.setServiceParent(module_service) ftp_service = internet.TCPServer(settings.ftp_server_port, ftp.FTPFactory(transfer_portal), interface=settings.ftp_server_ip) ftp_service.setServiceParent(module_service) return module_service
def startService(self): consumer = QueueConsumer( consumer_queue=ns.MAP_JOBS_QUEUE, pool_size=ns.MAX_MESSAGES, spec_path=ns.SPEC_PATH, vhost=ns.RABBITMQ_VHOST, username=ns.RABBITMQ_USERNAME, password=ns.RABBITMQ_PASSWORD, host=ns.RABBITMQ_HOST, port=ns.RABBITMQ_PORT ) publisher = QueuePublisher( declare_strategy=1, spec_path=ns.SPEC_PATH, vhost=ns.RABBITMQ_VHOST, username=ns.RABBITMQ_USERNAME, password=ns.RABBITMQ_PASSWORD, host=ns.RABBITMQ_HOST, port=ns.RABBITMQ_PORT ) hosts = self.options.get('hosts') mongo_pool = {} for host in hosts: mongo_pool[host] = txmongo.lazyMongoConnectionPool(\ host=host, port=ns.MONGODB_PORT) ReduceWorker(consumer, publisher, mongo_pool)
def startService(self): consumer = QueueConsumer( consumer_queue=ns.MAP_JOBS_QUEUE, pool_size=ns.MAX_MESSAGES, spec_path=ns.SPEC_PATH, vhost=ns.RABBITMQ_VHOST, username=ns.RABBITMQ_USERNAME, password=ns.RABBITMQ_PASSWORD, host=ns.RABBITMQ_HOST, port=ns.RABBITMQ_PORT ) publisher = QueuePublisher( declare_strategy=1, spec_path=ns.SPEC_PATH, vhost=ns.RABBITMQ_VHOST, username=ns.RABBITMQ_USERNAME, password=ns.RABBITMQ_PASSWORD, host=ns.RABBITMQ_HOST, port=ns.RABBITMQ_PORT ) # gettings mongo db collection pool _db = txmongo.lazyMongoConnectionPool(\ host=ns.MONGODB_HOST, port=ns.MONGODB_PORT) MapWorker(consumer, publisher, _db[ns.DB_NAME])
def __init__(self): mongodb = txmongo.lazyMongoConnectionPool() handlers = [ (r"/", IndexHandler, dict(mongodb=mongodb)), (r"/createuser", CreateUserHandler, dict(mongodb=mongodb)), ] cyclone.web.Application.__init__(self, handlers, debug=True)
def get_mongo_connection(): """ :return: Global MongoDB connection (adpay.db.MONGO_CONNECTION) """ global MONGO_CONNECTION if MONGO_CONNECTION is None: MONGO_CONNECTION = yield txmongo.lazyMongoConnectionPool( port=db_const.MONGO_DB_PORT) defer.returnValue(MONGO_CONNECTION)
def __init__(self): handlers = [ (r"/(.*)", MainHandler), ] settings = dict( cookie_secret="32oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", mongo=txmongo.lazyMongoConnectionPool(), redis=txredisapi.lazyRedisConnectionPool(), template_path=os.path.join(os.path.dirname(__file__), "templates"), ) cyclone.web.Application.__init__(self, handlers, **settings)
def __init__(self): handlers = [ (r"/", MainHandler), (r"/auth/login", LoginHandler), (r"/auth/logout", LogoutHandler), ] settings = dict( login_url="/auth/login", cookie_secret="32oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", mongo=txmongo.lazyMongoConnectionPool(), ) cyclone.web.Application.__init__(self, handlers, **settings)
def startService(self): service.Service.startService(self) self.http_factory.mongo = lazyMongoConnectionPool( host=self.settings.mongo.host, port=self.settings.mongo.port, ) reactor.listenTCP( port=self.settings.http.port, factory=self.http_factory, interface=self.settings.http.interface, )
def setup(cls, url=None, log=None): if cls.mongo_conn is None: if url is None: url = conf.MONGO_URL hostname, port, cls.mongo_db, username, password = url_parse( url, 'mongodb') AuthMongoProtocol.database = cls.mongo_db AuthMongoProtocol.username = username AuthMongoProtocol.password = password if log is None: log = getLoggerAdapter(getLogger(__name__), id='MONGO_CONNECTIONPOOL') cls.log = AuthMongoProtocol.log = log txmongo._MongoFactory.protocol = AuthMongoProtocol txmongo._MongoFactory.maxRetries = conf.BACKEND_MAX_RETRIES txmongo._MongoFactory.maxDelay = conf.BACKEND_MAX_DELAY cls.mongo_conn = txmongo.lazyMongoConnectionPool( host=hostname, port=port, reconnect=True, pool_size=conf.MONGO_POOL)
def applicationRegistered(cls, server): """ Called when the application is first registered (imported) into the server. *server* (``twisted.spread.pb.Root``) is the server root instance. """ config_filepath = cls.config_filepath if not os.path.exists(config_filepath): raise OSError(errno.ENOENT, "Config %r does not exist." % config_filepath, config_filepath) index_dirpath = cls.index_dirpath if not os.path.exists(index_dirpath): raise OSError(errno.ENOENT, "Index %r does not exist." % index_dirpath, index_dirpath) elif not os.path.isdir(index_dirpath): raise OSError(errno.ENOTDIR, "Index %r is not a directory." % index_dirpath, index_dirpath) # Read config. with open(config_filepath, 'rb') as fh: config = json.load(fh) cls.config = config # Connect to mongo. host = config['mongo']['host'] port = config['mongo'].get('port', None) or 27017 thread_pool = reactor.getThreadPool() pool_size = int(math.ceil((thread_pool.min + thread_pool.max) / 2)) cls.mongo = txmongo.lazyMongoConnectionPool(host=host, port=port, pool_size=pool_size) cls.order_db = cls.mongo[config['mongo']['order_dbname']] cls.order_tb = cls.order_db[config['mongo']['order_tbname']] # Initialize PyLucene. lucene.initVM() # Open index. index_dir = lucene.NIOFSDirectory(lucene.File(_index_dirpath)) #index_dir = lucene.SimpleFSDirectory(lucene.File(_index_dirpath)) cls.searcher = lucene.IndexSearcher(index_dir) print "REGISTERED %r" % cls
def startService(self): consumer = QueueConsumer(consumer_queue=ns.MAP_JOBS_QUEUE, pool_size=ns.MAX_MESSAGES, spec_path=ns.SPEC_PATH, vhost=ns.RABBITMQ_VHOST, username=ns.RABBITMQ_USERNAME, password=ns.RABBITMQ_PASSWORD, host=ns.RABBITMQ_HOST, port=ns.RABBITMQ_PORT) publisher = QueuePublisher(declare_strategy=1, spec_path=ns.SPEC_PATH, vhost=ns.RABBITMQ_VHOST, username=ns.RABBITMQ_USERNAME, password=ns.RABBITMQ_PASSWORD, host=ns.RABBITMQ_HOST, port=ns.RABBITMQ_PORT) # gettings mongo db collection pool _db = txmongo.lazyMongoConnectionPool(\ host=ns.MONGODB_HOST, port=ns.MONGODB_PORT) MapWorker(consumer, publisher, _db[ns.DB_NAME])
#!/usr/bin/python # coding: utf-8 from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks from txmongo import lazyMongoConnectionPool db = lazyMongoConnectionPool("127.0.0.1", 27017, pool_size=16) collection = db.twisted.asdf def res(result): print(result) reactor.stop() @inlineCallbacks def run_me(): # d = collection.insert({"first document": None}, safe=True) # d.addCallback(lambda ignored: reactor.stop()) result = yield collection.find({}, fields={"_id": 0}) print(result) reactor.stop() d = collection.insert(...) d.addCallback(res) # d.addCallback(res) reactor.callLater(0, run_me) reactor.run()
def __init__(self): _db = txmongo.lazyMongoConnectionPool() self.providers = _db.evm.providers self.domains = _db.evm.domains
# encoding: utf-8 from twisted.internet import reactor from twisted.web import server, resource from twisted.web.server import NOT_DONE_YET import txmongo # import sys # from twisted.python import log # log.startLogging(sys.stdout) db = txmongo.lazyMongoConnectionPool().perftest.messages class Root(resource.Resource): isLeaf = True def _success(self, value, request): counter = 0 for m in value: counter += len(m['longStringAttribute']) request.write('200 OK. Char counter = %i' % counter) request.finish() def _failure(self, error, request): request.setResponseCode(404) request.write("Invalid request") request.finish() def render_GET(self, request): id = int(request.uri.rsplit('/', 1)[-1]) d = db.find({"_id": id}, limit=50)
from twisted.web.resource import Resource import bcrypt from twisted.web.server import NOT_DONE_YET import txmongo from txmongo import filter as _filter import cgi import datetime from txmongo._pymongo.objectid import ObjectId from twisted.web.static import File from atlas.template import render_response _client = txmongo.lazyMongoConnectionPool() _db = _client.atlas sessions = set() class PostResource(Resource): def __init__(self, post_id): Resource.__init__(self) self.post_id = post_id def render_GET(self, request): def handle_post(post): context = {'post': post} request.write(render_response('singlepost.html', context)) request.finish() _id = ObjectId(self.post_id) deferred = _db.posts.find_one(_id) deferred.addCallback(handle_post) return NOT_DONE_YET
def __init__(self): self.config = Config("servrhe.json", { # Bot config "password": "", "channels": ["#commie-subs","#commie-staff"], "notifies": {}, "premux_dir": "", "mad": False, "markov_banned": ["c","k","kb","sync","op","deop","protect","deprotect","ban","unban"], # Release config "rip_host": "", "cr_user": "", "cr_pass": "", "funi_user": "", "funi_pass": "", "ftp_host": "", "ftp_port": 21, "ftp_user": "", "ftp_pass": "", "ftp_encode_dir": "", "xdcc_host": "", "xdcc_port": 21, "xdcc_user": "", "xdcc_pass": "", "xdcc_folder": "", "seed_host": "", "seed_port": 21, "seed_user": "", "seed_pass": "", "seed_file_folder": "", "seed_torrent_folder": "", "nyaa_user": "", "nyaa_pass": "", "tt_user": "", "tt_pass": "", "blog_user": "", "blog_pass": "", "mal_user": "", "mal_pass": "", # Showtime config "key": "", "base": "http://commie.milkteafuzz.com/st", "positions": ["translator","editor","typesetter","timer","encoding"], # Topic config "topic": ["☭ Commie Subs ☭",20,20.56], # Database "db_host": "", "db_port": "", "db_database": "" }) self.pluginmanager = PluginManager("commands") self.db = getattr(txmongo.lazyMongoConnectionPool(self.config.db_host, self.config.db_port, pool_size=3), self.config.db_database) self.alias = Aliases("alias.json") self.markov = Markov(self.db, self.alias) self.crunchy = Crunchyroll("crunchyroll.json", self.config, self.broadcast) self.funi = Funimation("funimation.json", self.config, self.broadcast) reactor.addSystemEventTrigger("before", "shutdown", self.shutdown) t = task.LoopingCall(self.refresh_shows) t.start(5*60) # 5 minutes n = task.LoopingCall(self.check_notifies) n.start(30) # Every 30 seconds