コード例 #1
0
ファイル: test_messages.py プロジェクト: ozgurakan/marconi
    def setUp(self):
        super(MessagesBaseTest, self).setUp()

        self.wsgi_cfg = config.namespace(
            'drivers:transport:wsgi').from_options()

        self.project_id = '7e55e1a7e'
        self.queue_path = '/v1/queues/fizbit'

        doc = '{"_ttl": 60}'
        self.simulate_put(self.queue_path, self.project_id, body=doc)

        self.headers = {
            'Client-ID': '30387f00',
        }
コード例 #2
0
    def setUp(self):
        super(MessagesBaseTest, self).setUp()

        self.wsgi_cfg = config.namespace(
            'drivers:transport:wsgi').from_options()

        self.project_id = '7e55e1a7e'
        self.queue_path = '/v1/queues/fizbit'
        self.messages_path = self.queue_path + '/messages'

        doc = '{"_ttl": 60}'
        self.simulate_put(self.queue_path, self.project_id, body=doc)

        self.headers = {
            'Client-ID': '30387f00',
        }
コード例 #3
0
ファイル: test_claims.py プロジェクト: amitgandhinz/marconi
    def setUp(self):
        super(ClaimsBaseTest, self).setUp()

        self.wsgi_cfg = config.namespace(
            'drivers:transport:wsgi').from_options()

        self.project_id = '480924'
        self.queue_path = '/v1/queues/fizbit'
        self.claims_path = self.queue_path + '/claims'
        self.messages_path = self.queue_path + '/messages'

        doc = '{"_ttl": 60}'

        self.simulate_put(self.queue_path, self.project_id, body=doc)
        self.assertEquals(self.srmock.status, falcon.HTTP_201)

        doc = json.dumps([{'body': 239, 'ttl': 300}] * 10)
        self.simulate_post(self.queue_path + '/messages', self.project_id,
                           body=doc, headers={'Client-ID': '30387f00'})
        self.assertEquals(self.srmock.status, falcon.HTTP_201)
コード例 #4
0
ファイル: test_claims.py プロジェクト: sujala/marconi-1
    def setUp(self):
        super(ClaimsBaseTest, self).setUp()

        self.wsgi_cfg = config.namespace(
            'drivers:transport:wsgi').from_options()

        self.project_id = '480924'
        self.queue_path = '/v1/queues/fizbit'
        self.claims_path = self.queue_path + '/claims'
        self.messages_path = self.queue_path + '/messages'

        doc = '{"_ttl": 60}'

        self.simulate_put(self.queue_path, self.project_id, body=doc)
        self.assertEquals(self.srmock.status, falcon.HTTP_201)

        doc = json.dumps([{'body': 239, 'ttl': 300}] * 10)
        self.simulate_post(self.queue_path + '/messages', self.project_id,
                           body=doc, headers={'Client-ID': '30387f00'})
        self.assertEquals(self.srmock.status, falcon.HTTP_201)
コード例 #5
0
ファイル: driver.py プロジェクト: sujala/marconi-1
import marconi.openstack.common.log as logging
from marconi import transport
from marconi.transport import auth
from marconi.transport.wsgi import claims
from marconi.transport.wsgi import health
from marconi.transport.wsgi import messages
from marconi.transport.wsgi import metadata
from marconi.transport.wsgi import queues
from marconi.transport.wsgi import stats
from marconi.transport.wsgi import v1

OPTIONS = {'bind': '0.0.0.0', 'port': 8888}

PROJECT_CFG = config.project('marconi')
GLOBAL_CFG = PROJECT_CFG.from_options()
WSGI_CFG = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS)

LOG = logging.getLogger(__name__)


def _check_media_type(req, resp, params):
    if not req.client_accepts('application/json'):
        raise falcon.HTTPNotAcceptable(
            u'''
Endpoint only serves `application/json`; specify client-side
media type support with the "Accept" header.''',
            href=u'http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html',
            href_text=u'14.1 Accept, Hypertext Transfer Protocol -- HTTP/1.1')


def _extract_project_id(req, resp, params):
コード例 #6
0
ファイル: claims.py プロジェクト: pombredanne/marconi
import falcon

from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils


LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
    metadata_max_length=64 * 1024
)

CLAIM_POST_SPEC = (('ttl', int), ('grace', int))
CLAIM_PATCH_SPEC = (('ttl', int),)


class CollectionResource(object):

    __slots__ = ('claim_controller')

    def __init__(self, claim_controller):
        self.claim_controller = claim_controller

    def on_post(self, req, resp, project_id, queue_name):
        LOG.debug(_(u'Claims collection POST - queue: %(queue)s, '
コード例 #7
0
    def setUp(self):
        if not os.environ.get("MONGODB_TEST_LIVE"):
            self.skipTest("No MongoDB instance running")
        super(QueueLifecycleMongoDBTests, self).setUp()

        self.cfg = config.namespace("drivers:storage:mongodb").from_options()
コード例 #8
0
ファイル: driver.py プロジェクト: racker/marconi
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import falcon

from marconi.common import config
from marconi import transport


cfg = config.namespace("drivers:transport:wsgi").from_options(port=8888)


class Driver(transport.DriverBase):
    def __init__(self, queue_controller, message_controller, claim_controller):

        queues = transport.wsgi.QueuesResource(queue_controller)

        self.app = api = falcon.API()
        api.add_route("/v1/{tenant_id}/queues/{queue_name}", queues)

    def listen(self):
        pass
コード例 #9
0
ファイル: claims.py プロジェクト: amitgandhinz/marconi
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from marconi.common import config
from marconi.queues.storage import base
from marconi.queues.storage import exceptions
from marconi.queues.storage.sqlite import utils

CFG = config.namespace('limits:storage').from_options(
    default_message_paging=10,
)


class ClaimController(base.ClaimBase):
    def __init__(self, driver):
        self.driver = driver
        self.driver.run('''
            create table
            if not exists
            Claims (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                qid INTEGER,
                ttl INTEGER,
                created DATETIME,  -- seconds since the Julian day
                FOREIGN KEY(qid) references Queues(id) on delete cascade
コード例 #10
0
ファイル: test_claims.py プロジェクト: sujala/marconi-1
    def setUp(self):
        if not os.environ.get('MONGODB_TEST_LIVE'):
            self.skipTest('No MongoDB instance running')
        super(ClaimsMongoDBTests, self).setUp()

        self.cfg = config.namespace('drivers:storage:mongodb').from_options()
コード例 #11
0
ファイル: test_config.py プロジェクト: sujala/marconi-1
 def test_wrong_type(self):
     ns = config.namespace('local')
     with testing.expect(config.cfg.Error):
         ns.from_options(opt={})
コード例 #12
0
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from stevedore import driver

from marconi.common import config
from marconi.common import decorators
from marconi.common import exceptions
from marconi.openstack.common import log
from marconi import transport  # NOQA


PROJECT_CFG = config.project('marconi')
CFG = config.namespace('drivers').from_options(
    transport='wsgi',
    storage='sqlite')

LOG = log.getLogger(__name__)


class Bootstrap(object):
    """Defines the Marconi bootstrapper.

    The bootstrap loads up drivers per a given configuration, and
    manages their lifetimes.
    """

    def __init__(self, config_file=None, cli_args=None):
        PROJECT_CFG.load(filename=config_file, args=cli_args)
        log.setup('marconi')
コード例 #13
0
    def setUp(self):
        super(QueueLifecycleBaseTest, self).setUp()

        self.wsgi_cfg = config.namespace(
            'drivers:transport:wsgi').from_options()
コード例 #14
0
ファイル: test_config.py プロジェクト: TheSriram/marconi
 def test_wrong_type(self):
     ns = config.namespace('local')
     with testtools.ExpectedException(config.cfg.Error):
         ns.from_options(opt={})
コード例 #15
0
ファイル: messages.py プロジェクト: kgriffs/marconi
# limitations under the License.
import falcon

from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils


LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
    content_max_length=256 * 1024
)

MESSAGE_POST_SPEC = (('ttl', int), ('body', '*'))


class CollectionResource(object):

    __slots__ = ('message_controller')

    def __init__(self, message_controller):
        self.message_controller = message_controller

    #-----------------------------------------------------------------------
    # Helpers
    #-----------------------------------------------------------------------
コード例 #16
0
    field names will be, most of the time, the first
    letter of their long name.
"""

import pymongo.errors

from marconi.common import config
import marconi.openstack.common.log as logging
from marconi.openstack.common import timeutils
from marconi import storage
from marconi.storage import exceptions
from marconi.storage.mongodb import utils

LOG = logging.getLogger(__name__)
CFG = config.namespace('limits:storage').from_options(
    default_queue_paging=10,
)


class QueueController(storage.QueueBase):
    """Implements queue resource operations using MongoDB.

    Queues:
        Name         Field
        ------------------
        name        ->   n
        project     ->   p
        counter     ->   c
        metadata    ->   m

    """
コード例 #17
0
    'database': 'marconi',

    # Maximum number of times to retry a failed operation. Currently
    # only used for retrying a message post.
    'max_attempts': 1000,

    # Maximum sleep interval between retries (actual sleep time
    # increases linearly according to number of attempts performed).
    'max_retry_sleep': 0.1,

    # Maximum jitter interval, to be added to the sleep interval, in
    # order to decrease probability that parallel requests will retry
    # at the same instant.
    'max_retry_jitter': 0.005,

    # Frequency of message garbage collections, in seconds
    'gc_interval': 5 * 60,

    # Threshold of number of expired messages to reach in a given
    # queue, before performing the GC. Useful for reducing frequent
    # locks on the DB for non-busy queues, or for worker queues
    # which process jobs quickly enough to keep the number of in-
    # flight messages low.
    #
    # Note: The higher this number, the larger the memory-mapped DB
    # files will be.
    'gc_threshold': 1000,
}

CFG = config.namespace('drivers:storage:mongodb').from_options(**OPTIONS)
コード例 #18
0
ファイル: metadata.py プロジェクト: sujala/marconi-1
import falcon

from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils


LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
    metadata_max_length=64 * 1024
)


class Resource(object):
    __slots__ = ('queue_ctrl', )

    def __init__(self, queue_controller):
        self.queue_ctrl = queue_controller

    def on_get(self, req, resp, project_id, queue_name):
        LOG.debug(_(u'Queue metadata GET - queue: %(queue)s, '
                    u'project: %(project)s') %
                  {'queue': queue_name, 'project': project_id})

        try:
コード例 #19
0
 def test_wrong_type(self):
     ns = config.namespace('local')
     with testtools.ExpectedException(config.cfg.Error):
         ns.from_options(opt={})
コード例 #20
0
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import time

from marconi.common import config
from marconi import storage
from marconi.storage import mongodb
from marconi.storage.mongodb import controllers
from marconi.tests.storage import base
from marconi.tests import util as testing


cfg = config.namespace("drivers:storage:mongodb").from_options()


class MongodbDriverTest(testing.TestBase):

    def setUp(self):
        if not os.environ.get("MONGODB_TEST_LIVE"):
            self.skipTest("No MongoDB instance running")

        super(MongodbDriverTest, self).setUp()
        self.load_conf("wsgi_mongodb.conf")

    def test_db_instance(self):
        driver = mongodb.Driver()
        db = driver.db
        self.assertEquals(db.name, cfg.database)
コード例 #21
0
ファイル: bootstrap.py プロジェクト: jasonjohnson/marconi
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from marconi.common import config
from marconi.common import exceptions
from marconi.openstack.common import importutils


cfg_handle = config.project('marconi')
cfg = config.namespace('drivers').from_options(
    transport='marconi.transport.wsgi',
    storage='marconi.storage.sqlite')


class Bootstrap(object):
    """
    Defines the Marconi Bootstrap

    The bootstrap loads up drivers per a given configuration, and manages their
    lifetimes.
    """

    def __init__(self, config_file=None, cli_args=None):
        cfg_handle.load(filename=config_file, args=cli_args)

        self.storage_module = import_driver(cfg.storage)
コード例 #22
0
ファイル: driver.py プロジェクト: BryanSD/marconi
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import contextlib
import sqlite3

import msgpack

from marconi.common import config
from marconi import storage
from marconi.storage.sqlite import controllers
from marconi.storage.sqlite import utils

CFG = config.namespace('drivers:storage:sqlite').from_options(
    database=':memory:')


class Driver(storage.DriverBase):

    def __init__(self):
        self.__path = CFG.database
        self.__conn = sqlite3.connect(self.__path,
                                      detect_types=sqlite3.PARSE_DECLTYPES)
        self.__db = self.__conn.cursor()
        self.run('''PRAGMA foreign_keys = ON''')

    @staticmethod
    def pack(o):
        """Converts a Python variable to a custom SQlite `DOCUMENT`.
コード例 #23
0
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import sqlite3

from marconi.common import config
from marconi import storage
from marconi.storage import base
from marconi.storage import exceptions


cfg = config.namespace('drivers:storage:reference').from_options(
        database=':memory:')


class Driver(storage.DriverBase):
    def __init__(self):
        self.__path = cfg.database
        self.__conn = sqlite3.connect(self.__path)
        self.__db = self.__conn.cursor()

    def run(self, sql, *args):
        return self.__db.execute(sql, args)

    def get(self, sql, *args):
        return self.run(sql, *args).fetchone()

    def __enter__(self):
コード例 #24
0
    def setUp(self):
        super(QueueLifecycleBaseTest, self).setUp()

        self.wsgi_cfg = config.namespace(
            'drivers:transport:wsgi').from_options()
コード例 #25
0
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon

from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils

LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
    content_max_length=256 * 1024)

MESSAGE_POST_SPEC = (('ttl', int), ('body', '*'))


class CollectionResource(object):

    __slots__ = ('message_controller')

    def __init__(self, message_controller):
        self.message_controller = message_controller

    #-----------------------------------------------------------------------
    # Helpers
    #-----------------------------------------------------------------------
コード例 #26
0
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import falcon

from marconi.common import config
from marconi import transport

cfg = config.namespace('drivers:transport:wsgi').from_options(port=8888)


class Driver(transport.DriverBase):
    def __init__(self, queue_controller, message_controller, claim_controller):

        queues = transport.wsgi.QueuesResource(queue_controller)

        self.app = api = falcon.API()
        api.add_route('/v1/{tenant_id}/queues/{queue_name}', queues)

    def listen(self):
        pass
コード例 #27
0
ファイル: driver.py プロジェクト: BryanSD/marconi
import marconi.openstack.common.log as logging
from marconi import transport
from marconi.transport import auth
from marconi.transport.wsgi import claims
from marconi.transport.wsgi import messages
from marconi.transport.wsgi import queues
from marconi.transport.wsgi import stats

OPTIONS = {
    'bind': '0.0.0.0',
    'port': 8888
}

PROJECT_CFG = config.project('marconi')
GLOBAL_CFG = PROJECT_CFG.from_options()
WSGI_CFG = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS)

LOG = logging.getLogger(__name__)


def _extract_project_id(req, resp, params):
    params['project_id'] = req.get_header('X-PROJECT-ID')


class Driver(transport.DriverBase):

    def __init__(self, storage):
        super(Driver, self).__init__(storage)

        self.app = falcon.API(before=_extract_project_id)
コード例 #28
0
ファイル: test_claims.py プロジェクト: amitgandhinz/marconi
    def setUp(self):
        if not os.environ.get('MONGODB_TEST_LIVE'):
            self.skipTest('No MongoDB instance running')
        super(ClaimsMongoDBTests, self).setUp()

        self.cfg = config.namespace('drivers:storage:mongodb').from_options()
コード例 #29
0
ファイル: options.py プロジェクト: BryanSD/marconi
    'database': 'marconi',

    # Maximum number of times to retry a failed operation. Currently
    # only used for retrying a message post.
    'max_attempts': 1000,

    # Maximum sleep interval between retries (actual sleep time
    # increases linearly according to number of attempts performed).
    'max_retry_sleep': 0.1,

    # Maximum jitter interval, to be added to the sleep interval, in
    # order to decrease probability that parallel requests will retry
    # at the same instant.
    'max_retry_jitter': 0.005,

    # Frequency of message garbage collections, in seconds
    'gc_interval': 5 * 60,

    # Threshold of number of expired messages to reach in a given
    # queue, before performing the GC. Useful for reducing frequent
    # locks on the DB for non-busy queues, or for worker queues
    # which process jobs quickly enough to keep the number of in-
    # flight messages low.
    #
    # Note: The higher this number, the larger the memory-mapped DB
    # files will be.
    'gc_threshold': 1000,
}

CFG = config.namespace('drivers:storage:mongodb').from_options(**OPTIONS)
コード例 #30
0
ファイル: test_config.py プロジェクト: BryanSD/marconi
 def test_wrong_type(self):
     ns = config.namespace('local')
     with testing.expect(config.cfg.Error):
         ns.from_options(opt={})
コード例 #31
0
ファイル: driver.py プロジェクト: sujala/marconi-1
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import contextlib
import sqlite3

import msgpack

from marconi.common import config
from marconi import storage
from marconi.storage.sqlite import controllers
from marconi.storage.sqlite import utils

CFG = config.namespace('drivers:storage:sqlite').from_options(
    database=':memory:')


class Driver(storage.DriverBase):
    def __init__(self):
        self.__path = CFG.database
        self.__conn = sqlite3.connect(self.__path,
                                      detect_types=sqlite3.PARSE_DECLTYPES)
        self.__db = self.__conn.cursor()
        self.run('''PRAGMA foreign_keys = ON''')

    @staticmethod
    def pack(o):
        """Converts a Python variable to a custom SQlite `DOCUMENT`.

        :param o: a Python str, unicode, int, long, float, bool, None
コード例 #32
0
import simplejson as json

from marconi.common import config
from marconi.common import exceptions

OPTIONS = {
    'queue_paging_uplimit': 20,
    'metadata_size_uplimit': 64 * 1024,
    'message_paging_uplimit': 20,
    'message_size_uplimit': 256 * 1024,
    'message_ttl_max': 1209600,
    'claim_ttl_max': 43200,
    'claim_grace_max': 43200,
}

CFG = config.namespace('limits:transport').from_options(**OPTIONS)

QUEUE_NAME_REGEX = re.compile('^[\w-]+$')


def queue_creation(name):
    """Restrictions on a queue name.

    :param name: The queue name
    :raises: ValidationFailed if the name is longer than 64 bytes or
        contains bytes other than ASCII digits, letters, underscore,
        and dash.
    """

    if len(name) > 64:
        raise exceptions.ValidationFailed(
コード例 #33
0
ファイル: base.py プロジェクト: jasonjohnson/marconi
Mongodb storage driver implementation
"""

import pymongo
import pymongo.errors

from marconi.common import config
from marconi import storage
from marconi.storage.mongodb import controllers

options = {
    "uri": None,
    "database": "marconi",
}

cfg = config.namespace('drivers:storage:mongodb').from_options(**options)


class Driver(storage.DriverBase):

    def __init__(self):
        self._database = None

    @property
    def db(self):
        """
        Property for lazy instantiation of
        mongodb's database.
        """
        if not self._database:
            if cfg.uri and 'replicaSet' in cfg.uri: