def __init__(self):
settings = dict(
cookie_secret="2c-*JKf74*41pfgj8.J|0G\Iuv@H:!,x]%V",
template_path=os.path.join(os.path.dirname(__file__), "templates"),
static_path=os.path.join(os.path.dirname(__file__), "static"),
xsrf_cookies=False,
autoescape=None
)
mongo_uri = os.environ.get('MONGOLAB_URI', False)
if mongo_uri:
regex = re.compile("^(?P<scheme>.*):\/\/(?P<user>.*):(?P<password>.*)@(?P<host>.*):(?P<port>\d*)\/(?P<database>.*)$")
creds = regex.search(mongo_uri).groupdict()
mongo = txmongo.lazyMongoConnectionPool(host=creds['host'], port=int(creds['port']))
db = mongo[creds['database']]
db.authenticate(creds['user'], creds['password'])
else:
log.msg("Running locally, connectiong to local mongodb without authentication.")
mongo = txmongo.lazyMongoConnectionPool()
db = mongo['valldb']
handlers = [
(r"/(.*)/scripts.js", JSHandler),
(r"/(.*)", VideowallHandler, dict(db=db)),
(r"/static/(.*)", cyclone.web.StaticFileHandler,
dict(path=settings['static_path'])),
]
cyclone.web.Application.__init__(self, handlers, **settings)
def test_lazyMongoConnectionPool(self):
# lazyMongoConnection returns MongoAPI
rapi = txmongo.lazyMongoConnectionPool(mongo_host, mongo_port, pool_size=2)
self.assertEqual(isinstance(rapi, txmongo.MongoAPI), True)
yield rapi._connected
disconnected = yield rapi.disconnect()
self.assertEqual(disconnected, True)
def _init_search(cls):
"""
Initializes everything needed for search.
"""
config_path = cls.search_config_path
if not os.path.exists(config_path):
raise OSError(errno.ENOENT, "Config %r does not exist." % config_path, config_path)
config_dir = os.path.dirname(config_path)
# Read config.
with open(config_path, 'rb') as fh:
config = json.load(fh)
cls.search_config = config
# Connect to mongo.
host = config['mongo']['host']
port = config['mongo'].get('port', None) or 27017
thread_pool = reactor.getThreadPool()
pool_size = int(math.ceil((thread_pool.min + thread_pool.max) / 2))
cls.search_mongo = txmongo.lazyMongoConnectionPool(host=host, port=port, pool_size=pool_size)
cls.search_order_db = cls.search_mongo[config['mongo']['order_dbname']]
cls.search_order_tb = cls.search_order_db[config['mongo']['order_tbname']]
# Initialize PyLucene.
lucene.initVM()
# Open index.
index_path = os.path.abspath(os.path.join(config_dir, config['lucene']['index_path']))
if not os.path.exists(index_path):
raise OSError(errno.ENOENT, "Index %r does not exist." % index_path, index_path)
elif not os.path.isdir(index_path):
raise OSError(errno.ENOTDIR, "Index %r is not a directory." % index_path, index_path)
index_dir = lucene.NIOFSDirectory(lucene.File(index_path))
#index_dir = lucene.SimpleFSDirectory(lucene.File(index_path)) # windows
cls.search_searcher = lucene.IndexSearcher(index_dir)
def setup():
_db = txmongo.lazyMongoConnectionPool(MONGO_SERVER, 27017)
collection = _db.houseprices.houses
root = resource.Resource()
# Can not do a 'distinct' query with async mongo - so use a sync connection for this
sync_connection = pymongo.Connection(MONGO_SERVER, 27017, max_pool_size=10)
sync_db = sync_connection ["houseprices"]
dates = sync_db.houses.distinct( 'dateadded')
legal_postcodes_results = sync_db.houses.distinct( 'postcode_part')
postcodes = []
for p in legal_postcodes_results:
postcodes.append(p)
postcodes.sort()
sync_connection.close()
#root = ElementResource(db)
#root.putChild("/favicon.ico", FaviconHandler)
root.putChild("/", PriceElementResource(dates, postcodes, collection))
root.putChild("price", PriceElementResource(dates, postcodes, collection))
root.putChild("postcode", PostcodeElementResource(dates, postcodes, collection))
root.putChild("single", SingleElementResource(dates, collection))
root.putChild("hello", HelloHandler())
factory = Site(root)
reactor.listenTCP(8881, factory)
reactor.run()
def makeService(self, options):
# MongoDB
mongo.connection.set(txmongo.lazyMongoConnectionPool(
**settings.mongodb['connection']))
module_service = super(ArchiverMaker, self).makeService(options)
transfer_portal = portal.Portal(self.ftp_realm, [
checkers.HomeDirectoryExistence(settings.uploads_root,
settings.downloads_root)
])
##
#context = ssl.DefaultOpenSSLContextFactory(
# settings.server_key,
# settings.server_crt
#)
#
#ftp_service = internet.SSLServer(settings.ftp_server_port,
# ftp.FTPFactory(transfer_portal),
# context,
# interface=settings.ftp_server_ip)
##
conv_service = converter.ConversionService(settings.conv_server_port,
interface=settings.conv_server_ip)
conv_service.setServiceParent(module_service)
ftp_service = internet.TCPServer(settings.ftp_server_port,
ftp.FTPFactory(transfer_portal),
interface=settings.ftp_server_ip)
ftp_service.setServiceParent(module_service)
return module_service
def startService(self):
consumer = QueueConsumer(
consumer_queue=ns.MAP_JOBS_QUEUE,
pool_size=ns.MAX_MESSAGES,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT
)
publisher = QueuePublisher(
declare_strategy=1,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT
)
hosts = self.options.get('hosts')
mongo_pool = {}
for host in hosts:
mongo_pool[host] = txmongo.lazyMongoConnectionPool(\
host=host, port=ns.MONGODB_PORT)
ReduceWorker(consumer, publisher, mongo_pool)
def startService(self):
consumer = QueueConsumer(
consumer_queue=ns.MAP_JOBS_QUEUE,
pool_size=ns.MAX_MESSAGES,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT
)
publisher = QueuePublisher(
declare_strategy=1,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT
)
# gettings mongo db collection pool
_db = txmongo.lazyMongoConnectionPool(\
host=ns.MONGODB_HOST, port=ns.MONGODB_PORT)
MapWorker(consumer, publisher, _db[ns.DB_NAME])
def __init__(self):
mongodb = txmongo.lazyMongoConnectionPool()
handlers = [
(r"/", IndexHandler, dict(mongodb=mongodb)),
(r"/createuser", CreateUserHandler, dict(mongodb=mongodb)),
]
cyclone.web.Application.__init__(self, handlers, debug=True)
def __init__(self):
mongodb = txmongo.lazyMongoConnectionPool()
handlers = [
(r"/", IndexHandler, dict(mongodb=mongodb)),
(r"/createuser", CreateUserHandler, dict(mongodb=mongodb)),
]
cyclone.web.Application.__init__(self, handlers, debug=True)
def get_mongo_connection():
"""
:return: Global MongoDB connection (adpay.db.MONGO_CONNECTION)
"""
global MONGO_CONNECTION
if MONGO_CONNECTION is None:
MONGO_CONNECTION = yield txmongo.lazyMongoConnectionPool(
port=db_const.MONGO_DB_PORT)
defer.returnValue(MONGO_CONNECTION)
def __init__(self):
handlers = [
(r"/(.*)", MainHandler),
]
settings = dict(
cookie_secret="32oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
mongo=txmongo.lazyMongoConnectionPool(),
redis=txredisapi.lazyRedisConnectionPool(),
template_path=os.path.join(os.path.dirname(__file__), "templates"),
)
cyclone.web.Application.__init__(self, handlers, **settings)
def __init__(self):
handlers = [
(r"/", MainHandler),
(r"/auth/login", LoginHandler),
(r"/auth/logout", LogoutHandler),
]
settings = dict(
login_url="/auth/login",
cookie_secret="32oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
mongo=txmongo.lazyMongoConnectionPool(),
)
cyclone.web.Application.__init__(self, handlers, **settings)
def startService(self):
service.Service.startService(self)
self.http_factory.mongo = lazyMongoConnectionPool(
host=self.settings.mongo.host,
port=self.settings.mongo.port,
)
reactor.listenTCP(
port=self.settings.http.port,
factory=self.http_factory,
interface=self.settings.http.interface,
)
def startService(self):
service.Service.startService(self)
self.http_factory.mongo = lazyMongoConnectionPool(
host=self.settings.mongo.host,
port=self.settings.mongo.port,
)
reactor.listenTCP(
port=self.settings.http.port,
factory=self.http_factory,
interface=self.settings.http.interface,
)
def setup(cls, url=None, log=None):
if cls.mongo_conn is None:
if url is None:
url = conf.MONGO_URL
hostname, port, cls.mongo_db, username, password = url_parse(
url, 'mongodb')
AuthMongoProtocol.database = cls.mongo_db
AuthMongoProtocol.username = username
AuthMongoProtocol.password = password
if log is None:
log = getLoggerAdapter(getLogger(__name__),
id='MONGO_CONNECTIONPOOL')
cls.log = AuthMongoProtocol.log = log
txmongo._MongoFactory.protocol = AuthMongoProtocol
txmongo._MongoFactory.maxRetries = conf.BACKEND_MAX_RETRIES
txmongo._MongoFactory.maxDelay = conf.BACKEND_MAX_DELAY
cls.mongo_conn = txmongo.lazyMongoConnectionPool(
host=hostname,
port=port,
reconnect=True,
pool_size=conf.MONGO_POOL)
def applicationRegistered(cls, server):
"""
Called when the application is first registered (imported) into the
server.
*server* (``twisted.spread.pb.Root``) is the server root instance.
"""
config_filepath = cls.config_filepath
if not os.path.exists(config_filepath):
raise OSError(errno.ENOENT, "Config %r does not exist." % config_filepath, config_filepath)
index_dirpath = cls.index_dirpath
if not os.path.exists(index_dirpath):
raise OSError(errno.ENOENT, "Index %r does not exist." % index_dirpath, index_dirpath)
elif not os.path.isdir(index_dirpath):
raise OSError(errno.ENOTDIR, "Index %r is not a directory." % index_dirpath, index_dirpath)
# Read config.
with open(config_filepath, 'rb') as fh:
config = json.load(fh)
cls.config = config
# Connect to mongo.
host = config['mongo']['host']
port = config['mongo'].get('port', None) or 27017
thread_pool = reactor.getThreadPool()
pool_size = int(math.ceil((thread_pool.min + thread_pool.max) / 2))
cls.mongo = txmongo.lazyMongoConnectionPool(host=host, port=port, pool_size=pool_size)
cls.order_db = cls.mongo[config['mongo']['order_dbname']]
cls.order_tb = cls.order_db[config['mongo']['order_tbname']]
# Initialize PyLucene.
lucene.initVM()
# Open index.
index_dir = lucene.NIOFSDirectory(lucene.File(_index_dirpath))
#index_dir = lucene.SimpleFSDirectory(lucene.File(_index_dirpath))
cls.searcher = lucene.IndexSearcher(index_dir)
print "REGISTERED %r" % cls
def startService(self):
consumer = QueueConsumer(consumer_queue=ns.MAP_JOBS_QUEUE,
pool_size=ns.MAX_MESSAGES,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT)
publisher = QueuePublisher(declare_strategy=1,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT)
# gettings mongo db collection pool
_db = txmongo.lazyMongoConnectionPool(\
host=ns.MONGODB_HOST, port=ns.MONGODB_PORT)
MapWorker(consumer, publisher, _db[ns.DB_NAME])
#!/usr/bin/python
# coding: utf-8
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks
from txmongo import lazyMongoConnectionPool
db = lazyMongoConnectionPool("127.0.0.1", 27017, pool_size=16)
collection = db.twisted.asdf
def res(result):
print(result)
reactor.stop()
@inlineCallbacks
def run_me():
# d = collection.insert({"first document": None}, safe=True)
# d.addCallback(lambda ignored: reactor.stop())
result = yield collection.find({}, fields={"_id": 0})
print(result)
reactor.stop()
d = collection.insert(...)
d.addCallback(res)
# d.addCallback(res)
reactor.callLater(0, run_me)
reactor.run()
def __init__(self):
_db = txmongo.lazyMongoConnectionPool()
self.providers = _db.evm.providers
self.domains = _db.evm.domains
# encoding: utf-8
from twisted.internet import reactor
from twisted.web import server, resource
from twisted.web.server import NOT_DONE_YET
import txmongo
# import sys
# from twisted.python import log
# log.startLogging(sys.stdout)
db = txmongo.lazyMongoConnectionPool().perftest.messages
class Root(resource.Resource):
isLeaf = True
def _success(self, value, request):
counter = 0
for m in value:
counter += len(m['longStringAttribute'])
request.write('200 OK. Char counter = %i' % counter)
request.finish()
def _failure(self, error, request):
request.setResponseCode(404)
request.write("Invalid request")
request.finish()
def render_GET(self, request):
id = int(request.uri.rsplit('/', 1)[-1])
d = db.find({"_id": id}, limit=50)
from twisted.web.resource import Resource
import bcrypt
from twisted.web.server import NOT_DONE_YET
import txmongo
from txmongo import filter as _filter
import cgi
import datetime
from txmongo._pymongo.objectid import ObjectId
from twisted.web.static import File
from atlas.template import render_response
_client = txmongo.lazyMongoConnectionPool()
_db = _client.atlas
sessions = set()
class PostResource(Resource):
def __init__(self, post_id):
Resource.__init__(self)
self.post_id = post_id
def render_GET(self, request):
def handle_post(post):
context = {'post': post}
request.write(render_response('singlepost.html', context))
request.finish()
_id = ObjectId(self.post_id)
deferred = _db.posts.find_one(_id)
deferred.addCallback(handle_post)
return NOT_DONE_YET
def __init__(self):
self.config = Config("servrhe.json", {
# Bot config
"password": "",
"channels": ["#commie-subs","#commie-staff"],
"notifies": {},
"premux_dir": "",
"mad": False,
"markov_banned": ["c","k","kb","sync","op","deop","protect","deprotect","ban","unban"],
# Release config
"rip_host": "",
"cr_user": "",
"cr_pass": "",
"funi_user": "",
"funi_pass": "",
"ftp_host": "",
"ftp_port": 21,
"ftp_user": "",
"ftp_pass": "",
"ftp_encode_dir": "",
"xdcc_host": "",
"xdcc_port": 21,
"xdcc_user": "",
"xdcc_pass": "",
"xdcc_folder": "",
"seed_host": "",
"seed_port": 21,
"seed_user": "",
"seed_pass": "",
"seed_file_folder": "",
"seed_torrent_folder": "",
"nyaa_user": "",
"nyaa_pass": "",
"tt_user": "",
"tt_pass": "",
"blog_user": "",
"blog_pass": "",
"mal_user": "",
"mal_pass": "",
# Showtime config
"key": "",
"base": "http://commie.milkteafuzz.com/st",
"positions": ["translator","editor","typesetter","timer","encoding"],
# Topic config
"topic": ["☭ Commie Subs ☭",20,20.56],
# Database
"db_host": "",
"db_port": "",
"db_database": ""
})
self.pluginmanager = PluginManager("commands")
self.db = getattr(txmongo.lazyMongoConnectionPool(self.config.db_host, self.config.db_port, pool_size=3), self.config.db_database)
self.alias = Aliases("alias.json")
self.markov = Markov(self.db, self.alias)
self.crunchy = Crunchyroll("crunchyroll.json", self.config, self.broadcast)
self.funi = Funimation("funimation.json", self.config, self.broadcast)
reactor.addSystemEventTrigger("before", "shutdown", self.shutdown)
t = task.LoopingCall(self.refresh_shows)
t.start(5*60) # 5 minutes
n = task.LoopingCall(self.check_notifies)
n.start(30) # Every 30 seconds
