def main(config_ini):
print 'Starting up main'
_settings = configparser.SafeConfigParser()
with open(config_ini) as fp:
_settings.readfp(fp)
settings = dict(_settings.items('app:main'))
#formatting = dict(_settings.items('formatter_generic'))
logger_settings = dict(_settings.items('logger_uploader'))
settings.update(parse_asset_settings(settings, dev_ini_path=config_ini))
fh = RotatingFileHandler(
logger_settings.get('filename', 'uploader.log'),
maxBytes=int(logger_settings.get('maxBytes', 10000)),
encoding='utf-8'
)
lvl = logger_settings.get('level', 'DEBUG').upper()
lvl = logging.__getattribute__(lvl)
fh.setLevel(lvl)
formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s')
fh.setFormatter(formatter)
log.addHandler(fh)
log.setLevel(lvl)
url_worker = settings['zmq.worker_socket']
url_client = settings['zmq.socket']
context = zmq.Context(1)
# The 'frontend' facing the clients where new jobs are being sent
clients = context.socket(zmq.PULL)
clients.bind(url_client)
# The 'backend' facing the workers where received jobs are being pushed
workers = context.socket(zmq.PUSH)
workers.bind(url_worker)
conn = pymongo.Connection(
host=settings['db.mongo.host'],
port=int(settings['db.mongo.port'])
)
db = conn[settings['db.mongo.collection_name']]
cloud = None
if not settings['store_locally']:
log.info('Initializing cloud connection...')
cloud = settings['service'](**settings)
cloud.connect()
save_path = settings['save_path']
for i in range(int(settings['zmq.workers'])):
worker = UploadWorker(
name='[worker-thread-{}]'.format(i),
imager=ImageProcessor(
db=db,
image_save_path=settings['save_path']
),
context=context,
worker_url=url_worker,
db=db,
cloud=cloud,
save_path=save_path
)
worker.start()
try:
log.info('Starting zmq streamer')
log.info('Now waiting for jobs...')
zmq.device(zmq.STREAMER, clients, workers)
except KeyboardInterrupt:
pass
# We never get here... but if we do, shut down!
log.info('Shutting down...')
clients.close()
workers.close()
context.term()
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
authn_policy = AuthTktAuthenticationPolicy(
'supersecretstring',
callback=groupfinder
)
authz_policy = ACLAuthorizationPolicy()
session_factory = session_factory_from_settings(settings)
config = Configurator(
settings=settings,
authentication_policy=authn_policy,
authorization_policy=authz_policy,
session_factory=session_factory,
root_factory='m4ed.factories:RootFactory'
)
config.registry.settings['assets'] = parse_asset_settings(settings)
if config.registry.settings['hogan.compile']:
subprocess.call([config.registry.settings['hogan.compile']])
# Set up the mongo connection
mongo_host = config.registry.settings['db.mongo.host']
mongo_port = int(config.registry.settings['db.mongo.port'])
mongo_conn = pymongo.Connection(host=mongo_host, port=mongo_port)
config.registry.settings['db.mongo.conn'] = mongo_conn
config.set_request_property('m4ed.request_properties:db',
name='db', reify=True)
config.set_request_property('m4ed.request_properties.fs',
name='fs', reify=True)
# Set up redis connection
redis_host = config.registry.settings['db.redis.host']
redis_port = int(config.registry.settings['db.redis.port'])
redis_db_num = int(config.registry.settings['db.redis.db_num'])
config.registry.settings['db.redis.conn'] = redis.StrictRedis(
host=redis_host, port=redis_port, db=redis_db_num)
# Set up the shared math text parser
config.set_request_property('m4ed.request_properties:math_text_parser',
name='math_text_parser', reify=True)
config.set_request_property('m4ed.request_properties:user',
name='user', reify=True)
# For debugging matplotlib
# from matplotlib import verbose
# verbose.level = 'debug-annoying'
# Set up ZeroMQ work queue
work_queue = zmq.Context().socket(zmq.PUSH)
work_queue.connect(settings['zmq.socket'])
config.registry.settings['zmq.work_queue'] = work_queue
config.set_request_property('m4ed.request_properties:work_queue',
name='work_queue', reify=True)
config.include('pyramid_fanstatic')
config.include('m4ed.routes')
config.add_static_view('static', 'static', cache_max_age=3600)
json_renderer = JSON()
json_renderer.add_adapter(ObjectId, lambda obj, req: str(obj))
config.add_renderer('json', json_renderer)
config.scan()
return config.make_wsgi_app()