コード例 #1
0
ファイル: uploader.py プロジェクト: aape/m4ed
def main(config_ini):
    print 'Starting up main'
    _settings = configparser.SafeConfigParser()
    with open(config_ini) as fp:
        _settings.readfp(fp)
    settings = dict(_settings.items('app:main'))
    #formatting = dict(_settings.items('formatter_generic'))
    logger_settings = dict(_settings.items('logger_uploader'))
    settings.update(parse_asset_settings(settings, dev_ini_path=config_ini))

    fh = RotatingFileHandler(
        logger_settings.get('filename', 'uploader.log'),
        maxBytes=int(logger_settings.get('maxBytes', 10000)),
        encoding='utf-8'
        )

    lvl = logger_settings.get('level', 'DEBUG').upper()
    lvl = logging.__getattribute__(lvl)

    fh.setLevel(lvl)
    formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s')
    fh.setFormatter(formatter)

    log.addHandler(fh)
    log.setLevel(lvl)

    url_worker = settings['zmq.worker_socket']
    url_client = settings['zmq.socket']

    context = zmq.Context(1)

    # The 'frontend' facing the clients where new jobs are being sent
    clients = context.socket(zmq.PULL)
    clients.bind(url_client)

    # The 'backend' facing the workers where received jobs are being pushed
    workers = context.socket(zmq.PUSH)
    workers.bind(url_worker)

    conn = pymongo.Connection(
        host=settings['db.mongo.host'],
        port=int(settings['db.mongo.port'])
        )

    db = conn[settings['db.mongo.collection_name']]

    cloud = None
    if not settings['store_locally']:
        log.info('Initializing cloud connection...')
        cloud = settings['service'](**settings)
        cloud.connect()

    save_path = settings['save_path']

    for i in range(int(settings['zmq.workers'])):
        worker = UploadWorker(
            name='[worker-thread-{}]'.format(i),
            imager=ImageProcessor(
                db=db,
                image_save_path=settings['save_path']
                ),
            context=context,
            worker_url=url_worker,
            db=db,
            cloud=cloud,
            save_path=save_path
            )
        worker.start()

    try:
        log.info('Starting zmq streamer')
        log.info('Now waiting for jobs...')
        zmq.device(zmq.STREAMER, clients, workers)
    except KeyboardInterrupt:
        pass

    # We never get here... but if we do, shut down!
    log.info('Shutting down...')

    clients.close()
    workers.close()
    context.term()
コード例 #2
0
ファイル: __init__.py プロジェクト: aape/m4ed
def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """

    authn_policy = AuthTktAuthenticationPolicy(
        'supersecretstring',
        callback=groupfinder
    )
    authz_policy = ACLAuthorizationPolicy()

    session_factory = session_factory_from_settings(settings)

    config = Configurator(
        settings=settings,
        authentication_policy=authn_policy,
        authorization_policy=authz_policy,
        session_factory=session_factory,
        root_factory='m4ed.factories:RootFactory'
    )

    config.registry.settings['assets'] = parse_asset_settings(settings)

    if config.registry.settings['hogan.compile']:
        subprocess.call([config.registry.settings['hogan.compile']])

    # Set up the mongo connection
    mongo_host = config.registry.settings['db.mongo.host']
    mongo_port = int(config.registry.settings['db.mongo.port'])
    mongo_conn = pymongo.Connection(host=mongo_host, port=mongo_port)
    config.registry.settings['db.mongo.conn'] = mongo_conn
    config.set_request_property('m4ed.request_properties:db',
                                name='db', reify=True)
    config.set_request_property('m4ed.request_properties.fs',
                                name='fs', reify=True)

    # Set up redis connection
    redis_host = config.registry.settings['db.redis.host']
    redis_port = int(config.registry.settings['db.redis.port'])
    redis_db_num = int(config.registry.settings['db.redis.db_num'])
    config.registry.settings['db.redis.conn'] = redis.StrictRedis(
        host=redis_host, port=redis_port, db=redis_db_num)

    # Set up the shared math text parser
    config.set_request_property('m4ed.request_properties:math_text_parser',
                                name='math_text_parser', reify=True)

    config.set_request_property('m4ed.request_properties:user',
                                name='user', reify=True)

    # For debugging matplotlib
    # from matplotlib import verbose
    # verbose.level = 'debug-annoying'

    # Set up ZeroMQ work queue
    work_queue = zmq.Context().socket(zmq.PUSH)
    work_queue.connect(settings['zmq.socket'])
    config.registry.settings['zmq.work_queue'] = work_queue
    config.set_request_property('m4ed.request_properties:work_queue',
                                name='work_queue', reify=True)

    config.include('pyramid_fanstatic')
    config.include('m4ed.routes')
    config.add_static_view('static', 'static', cache_max_age=3600)

    json_renderer = JSON()
    json_renderer.add_adapter(ObjectId, lambda obj, req: str(obj))
    config.add_renderer('json', json_renderer)

    config.scan()
    return config.make_wsgi_app()