示例#1
0
def StartTabaAgentServer(server_endpoints, flush_period_seconds, port):
  """Initialize the Taba Agent, and setup Juno request handlers for the Taba
  Client and monitoring interface.

  Args:
    server_endpoints - List of Taba Server end-point dictionaries, containing
        'host' and 'port' fields for each end-point.
    flush_period_seconds - The period, in seconds, at which the Taba Agent
        should flush its buffer to the Server.
    port - Port the Taba Agent to listen to for receiving Events from Clients.
  """
  # Monkey patch gevent before doing anything.
  monkey.patch_all()

  # Initialize the Taba Agent
  global_taba_agent.Initialize(server_endpoints, flush_period_seconds)

  # Setup all necessary handlers.
  juno_patches.InitializeJuno('wsgi')

  _handlers = [
      misc_util.HandlerSpec('/post', HandlePost, methods=('post',)),
      misc_util.HandlerSpec('/status', HandleStatus),
  ]

  for h in _handlers:
    for method in h.methods:
      juno.route(h.route_url, method)(h.handler_func)

  # Move 'wsgi.input' to 'request_body_bytes' so that we can read it directly
  # instead of having Juno parse it into a dictionary.
  juno.config(
      'middleware',
      [('tellapart.taba.util.middleware.CircumventJunoInputParsing', {}),])

  application = juno.run()
  server_name = 'Taba Agent'

  from tellapart.taba.util import wsgi_server
  wsgi_server.launch_gevent_wsgi_server(application, port, 16, server_name)
示例#2
0
def StartTabaAgentServer(server_endpoints, flush_period_seconds, port):
    """Initialize the Taba Agent, and setup Juno request handlers for the Taba
  Client and monitoring interface.

  Args:
    server_endpoints - List of Taba Server end-point dictionaries, containing
        'host' and 'port' fields for each end-point.
    flush_period_seconds - The period, in seconds, at which the Taba Agent
        should flush its buffer to the Server.
    port - Port the Taba Agent to listen to for receiving Events from Clients.
  """
    # Monkey patch gevent before doing anything.
    monkey.patch_all()

    # Initialize the Taba Agent
    global_taba_agent.Initialize(server_endpoints, flush_period_seconds)

    # Setup all necessary handlers.
    juno_patches.InitializeJuno('wsgi')

    _handlers = [
        misc_util.HandlerSpec('/post', HandlePost, methods=('post', )),
        misc_util.HandlerSpec('/status', HandleStatus),
    ]

    for h in _handlers:
        for method in h.methods:
            juno.route(h.route_url, method)(h.handler_func)

    # Move 'wsgi.input' to 'request_body_bytes' so that we can read it directly
    # instead of having Juno parse it into a dictionary.
    juno.config('middleware', [
        ('tellapart.taba.util.middleware.CircumventJunoInputParsing', {}),
    ])

    application = juno.run()
    server_name = 'Taba Agent'

    from tellapart.taba.util import wsgi_server
    wsgi_server.launch_gevent_wsgi_server(application, port, 16, server_name)
示例#3
0
def StartTabaServer(
    primary_port,
    secondary_port,
    db_endpoints,
    db_vbuckets,
    server_endpoints,
    pruning_enabled=False,
    use_memory_engine=False):
  """Initialize the Taba Server, and setup Juno request handlers for the Taba
  Client and monitoring interface.

  Args:
    primary_port - Port to listen on for primary requests.
    secondary_port - Port to listen on for secondary requests.
    db_endpoints - List of database end-point information dictionaries. Each
        entry should contain 'host', 'port', 'vbuckets' (where 'vbuckets' is
        a 2 element list specifying the start and end vbucket for that
        end-point).
    db_vbuckets - Total number of vbuckets in the database.
    server_endpoints - List of Taba Server end-points information dictionaries.
        Each entry should contain 'host' and 'port' fields.
    pruning_enabled - If True, enable the periodic pruning service for this
        process.
    use_memory_engine - If True, use an in-memory database instead of the usual
        Redis one. Useful for testing.
  """
  # Monkey patch gevent before doing anything.
  from gevent import monkey
  monkey.patch_all()

  # Attempt to turn heapy remote monitoring on.
  try:
    import guppy.heapy.RM   #@UnusedImport
  except ImportError:
    pass

  # Attempt to attach a stack-trace dump to the SIGQUIT (kill -3) signal.
  try:
    import signal
    signal.signal(signal.SIGQUIT, _PrintTrace)
  except ImportError:
    pass

  # Compile the Cython components.
  from tellapart.taba.util.misc_util import BootstrapCython
  BootstrapCython(primary_port)

  from tellapart.frontend import juno_patches
  from tellapart.third_party import juno

  from tellapart.storage.engine import redis_engine
  from tellapart.taba import taba_client
  from tellapart.taba.server import taba_server
  from tellapart.taba.server import taba_server_storage_manager
  from tellapart.taba.server import taba_server_handlers
  from tellapart.taba.util.misc_util import HandlerSpec

  # Initialize the Taba Server, storage manager, and Redis engine.
  if use_memory_engine:
    from tellapart.storage.engine import memory_redis_engine
    engine = memory_redis_engine.MemoryRedisEngine(None, None)
    server_endpoints = None

    taba_url = 'http://localhost:%d/post' % secondary_port

  else:
    redis_endpoints = []
    for endpoint in db_endpoints:
      redis_endpoints.append(
          redis_engine.RedisServerEndpoint(
              host=endpoint['host'],
              port=endpoint['port'],
              vbucket_start=endpoint['vbuckets'][0],
              vbucket_end=endpoint['vbuckets'][1]))

    engine = redis_engine.RedisEngine(redis_endpoints, db_vbuckets)

    taba_url = 'http://localhost:%d/post' % server_endpoints[0]['port']

  dao = taba_server_storage_manager.TabaServerStorageManager(engine)
  taba_server_handlers.global_taba_server = \
      taba_server.TabaServer(dao, server_endpoints, prune=pruning_enabled)

  # Setup the local Taba Client.
  taba_client.Initialize('taba_server', taba_url, 60)

  # Setup all necessary handlers.
  juno_patches.InitializeJuno('wsgi')

  _handlers = [
      # Event posting handlers.
      HandlerSpec(
          '/post_zip',
          taba_server_handlers.HandlePostCompressed,
          methods=('POST',)),
      HandlerSpec(
          '/post',
          taba_server_handlers.HandlePostDirect,
          methods=('POST',)),

      # Single Get handlers.
      HandlerSpec('/raw', taba_server_handlers.HandleGetRaw),
      HandlerSpec('/projection', taba_server_handlers.HandleGetProjection),
      HandlerSpec('/aggregate', taba_server_handlers.HandleGetAggregate),
      HandlerSpec('/taba', taba_server_handlers.HandleGetTaba),

      # Batch Get handlers.
      HandlerSpec(
          '/raw_batch',
          taba_server_handlers.HandleGetRawBatch,
          methods=('POST',)),
      HandlerSpec(
          '/projection_batch',
          taba_server_handlers.HandleGetProjectionBatch,
          methods=('POST',)),
      HandlerSpec(
          '/aggregate_batch',
          taba_server_handlers.HandleGetAggretateBatch,
          methods=('POST',)),

      # Meta-data handlers.
      HandlerSpec('/clients', taba_server_handlers.HandleGetClients),
      HandlerSpec('/names', taba_server_handlers.HandleGetTabaNames),
      HandlerSpec('/type', taba_server_handlers.HandleGetType),

      # Administrative handlers.
      HandlerSpec('/delete', taba_server_handlers.HandleDeleteName),
      HandlerSpec('/prune', taba_server_handlers.HandlePrune),
      HandlerSpec('/upgrade', taba_server_handlers.HandleUpgrade),
      HandlerSpec('/status', taba_server_handlers.HandleStatus),
  ]

  for h in _handlers:
    for method in h.methods:
      juno.route(h.route_url, method)(h.handler_func)

  # Move 'wsgi.input' to 'request_body_bytes' so that we can read it directly
  # instead of having Juno parse it into a dictionary.
  juno.config(
      'middleware',
      [('tellapart.taba.util.middleware.CircumventJunoInputParsing', {}),])

  # Create and start the Primary and Secondary Server objects.
  from tellapart.taba.util import wsgi_server
  application = juno.run()

  primary_server = wsgi_server.launch_gevent_wsgi_server(
      application,
      port=primary_port,
      max_concurrent_requests=8,
      server_name='Taba Server - Primary',
      should_run_forever=False,
      use_clean_shutdown=True)

  secondary_server = wsgi_server.launch_gevent_wsgi_server(
      application,
      port=secondary_port,
      max_concurrent_requests=8,
      server_name='Taba Server - Secondary',
      should_run_forever=False,
      use_clean_shutdown=True)

  primary_server.start()
  secondary_server.start()

  primary_server._stopped_event.wait()
  secondary_server._stopped_event.wait()
示例#4
0
def StartTabaServer(primary_port,
                    secondary_port,
                    db_endpoints,
                    db_vbuckets,
                    server_endpoints,
                    pruning_enabled=False,
                    use_memory_engine=False):
    """Initialize the Taba Server, and setup Juno request handlers for the Taba
  Client and monitoring interface.

  Args:
    primary_port - Port to listen on for primary requests.
    secondary_port - Port to listen on for secondary requests.
    db_endpoints - List of database end-point information dictionaries. Each
        entry should contain 'host', 'port', 'vbuckets' (where 'vbuckets' is
        a 2 element list specifying the start and end vbucket for that
        end-point).
    db_vbuckets - Total number of vbuckets in the database.
    server_endpoints - List of Taba Server end-points information dictionaries.
        Each entry should contain 'host' and 'port' fields.
    pruning_enabled - If True, enable the periodic pruning service for this
        process.
    use_memory_engine - If True, use an in-memory database instead of the usual
        Redis one. Useful for testing.
  """
    # Monkey patch gevent before doing anything.
    from gevent import monkey
    monkey.patch_all()

    # Attempt to turn heapy remote monitoring on.
    try:
        import guppy.heapy.RM  #@UnusedImport
    except ImportError:
        pass

    # Attempt to attach a stack-trace dump to the SIGQUIT (kill -3) signal.
    try:
        import signal
        signal.signal(signal.SIGQUIT, _PrintTrace)
    except ImportError:
        pass

    # Compile the Cython components.
    from tellapart.taba.util.misc_util import BootstrapCython
    BootstrapCython(primary_port)

    from tellapart.frontend import juno_patches
    from tellapart.third_party import juno

    from tellapart.storage.engine import redis_engine
    from tellapart.taba import taba_client
    from tellapart.taba.server import taba_server
    from tellapart.taba.server import taba_server_storage_manager
    from tellapart.taba.server import taba_server_handlers
    from tellapart.taba.util.misc_util import HandlerSpec

    # Initialize the Taba Server, storage manager, and Redis engine.
    if use_memory_engine:
        from tellapart.storage.engine import memory_redis_engine
        engine = memory_redis_engine.MemoryRedisEngine(None, None)
        server_endpoints = None

        taba_url = 'http://localhost:%d/post' % secondary_port

    else:
        redis_endpoints = []
        for endpoint in db_endpoints:
            redis_endpoints.append(
                redis_engine.RedisServerEndpoint(
                    host=endpoint['host'],
                    port=endpoint['port'],
                    vbucket_start=endpoint['vbuckets'][0],
                    vbucket_end=endpoint['vbuckets'][1]))

        engine = redis_engine.RedisEngine(redis_endpoints, db_vbuckets)

        taba_url = 'http://localhost:%d/post' % server_endpoints[0]['port']

    dao = taba_server_storage_manager.TabaServerStorageManager(engine)
    taba_server_handlers.global_taba_server = \
        taba_server.TabaServer(dao, server_endpoints, prune=pruning_enabled)

    # Setup the local Taba Client.
    taba_client.Initialize('taba_server', taba_url, 60)

    # Setup all necessary handlers.
    juno_patches.InitializeJuno('wsgi')

    _handlers = [
        # Event posting handlers.
        HandlerSpec('/post_zip',
                    taba_server_handlers.HandlePostCompressed,
                    methods=('POST', )),
        HandlerSpec('/post',
                    taba_server_handlers.HandlePostDirect,
                    methods=('POST', )),

        # Single Get handlers.
        HandlerSpec('/raw', taba_server_handlers.HandleGetRaw),
        HandlerSpec('/projection', taba_server_handlers.HandleGetProjection),
        HandlerSpec('/aggregate', taba_server_handlers.HandleGetAggregate),
        HandlerSpec('/taba', taba_server_handlers.HandleGetTaba),

        # Batch Get handlers.
        HandlerSpec('/raw_batch',
                    taba_server_handlers.HandleGetRawBatch,
                    methods=('POST', )),
        HandlerSpec('/projection_batch',
                    taba_server_handlers.HandleGetProjectionBatch,
                    methods=('POST', )),
        HandlerSpec('/aggregate_batch',
                    taba_server_handlers.HandleGetAggretateBatch,
                    methods=('POST', )),

        # Meta-data handlers.
        HandlerSpec('/clients', taba_server_handlers.HandleGetClients),
        HandlerSpec('/names', taba_server_handlers.HandleGetTabaNames),
        HandlerSpec('/type', taba_server_handlers.HandleGetType),

        # Administrative handlers.
        HandlerSpec('/delete', taba_server_handlers.HandleDeleteName),
        HandlerSpec('/prune', taba_server_handlers.HandlePrune),
        HandlerSpec('/upgrade', taba_server_handlers.HandleUpgrade),
        HandlerSpec('/status', taba_server_handlers.HandleStatus),
    ]

    for h in _handlers:
        for method in h.methods:
            juno.route(h.route_url, method)(h.handler_func)

    # Move 'wsgi.input' to 'request_body_bytes' so that we can read it directly
    # instead of having Juno parse it into a dictionary.
    juno.config('middleware', [
        ('tellapart.taba.util.middleware.CircumventJunoInputParsing', {}),
    ])

    # Create and start the Primary and Secondary Server objects.
    from tellapart.taba.util import wsgi_server
    application = juno.run()

    primary_server = wsgi_server.launch_gevent_wsgi_server(
        application,
        port=primary_port,
        max_concurrent_requests=8,
        server_name='Taba Server - Primary',
        should_run_forever=False,
        use_clean_shutdown=True)

    secondary_server = wsgi_server.launch_gevent_wsgi_server(
        application,
        port=secondary_port,
        max_concurrent_requests=8,
        server_name='Taba Server - Secondary',
        should_run_forever=False,
        use_clean_shutdown=True)

    primary_server.start()
    secondary_server.start()

    primary_server._stopped_event.wait()
    secondary_server._stopped_event.wait()