def setUp(self):
super(MessagesBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
self.project_id = '7e55e1a7e'
self.queue_path = '/v1/queues/fizbit'
doc = '{"_ttl": 60}'
self.simulate_put(self.queue_path, self.project_id, body=doc)
self.headers = {
'Client-ID': '30387f00',
}
def setUp(self):
super(MessagesBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
self.project_id = '7e55e1a7e'
self.queue_path = '/v1/queues/fizbit'
self.messages_path = self.queue_path + '/messages'
doc = '{"_ttl": 60}'
self.simulate_put(self.queue_path, self.project_id, body=doc)
self.headers = {
'Client-ID': '30387f00',
}
def setUp(self):
super(ClaimsBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
self.project_id = '480924'
self.queue_path = '/v1/queues/fizbit'
self.claims_path = self.queue_path + '/claims'
self.messages_path = self.queue_path + '/messages'
doc = '{"_ttl": 60}'
self.simulate_put(self.queue_path, self.project_id, body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_201)
doc = json.dumps([{'body': 239, 'ttl': 300}] * 10)
self.simulate_post(self.queue_path + '/messages', self.project_id,
body=doc, headers={'Client-ID': '30387f00'})
self.assertEquals(self.srmock.status, falcon.HTTP_201)
def setUp(self):
super(ClaimsBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
self.project_id = '480924'
self.queue_path = '/v1/queues/fizbit'
self.claims_path = self.queue_path + '/claims'
self.messages_path = self.queue_path + '/messages'
doc = '{"_ttl": 60}'
self.simulate_put(self.queue_path, self.project_id, body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_201)
doc = json.dumps([{'body': 239, 'ttl': 300}] * 10)
self.simulate_post(self.queue_path + '/messages', self.project_id,
body=doc, headers={'Client-ID': '30387f00'})
self.assertEquals(self.srmock.status, falcon.HTTP_201)
import marconi.openstack.common.log as logging
from marconi import transport
from marconi.transport import auth
from marconi.transport.wsgi import claims
from marconi.transport.wsgi import health
from marconi.transport.wsgi import messages
from marconi.transport.wsgi import metadata
from marconi.transport.wsgi import queues
from marconi.transport.wsgi import stats
from marconi.transport.wsgi import v1
OPTIONS = {'bind': '0.0.0.0', 'port': 8888}
PROJECT_CFG = config.project('marconi')
GLOBAL_CFG = PROJECT_CFG.from_options()
WSGI_CFG = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS)
LOG = logging.getLogger(__name__)
def _check_media_type(req, resp, params):
if not req.client_accepts('application/json'):
raise falcon.HTTPNotAcceptable(
u'''
Endpoint only serves `application/json`; specify client-side
media type support with the "Accept" header.''',
href=u'http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html',
href_text=u'14.1 Accept, Hypertext Transfer Protocol -- HTTP/1.1')
def _extract_project_id(req, resp, params):
import falcon
from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils
LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
metadata_max_length=64 * 1024
)
CLAIM_POST_SPEC = (('ttl', int), ('grace', int))
CLAIM_PATCH_SPEC = (('ttl', int),)
class CollectionResource(object):
__slots__ = ('claim_controller')
def __init__(self, claim_controller):
self.claim_controller = claim_controller
def on_post(self, req, resp, project_id, queue_name):
LOG.debug(_(u'Claims collection POST - queue: %(queue)s, '
def setUp(self):
if not os.environ.get("MONGODB_TEST_LIVE"):
self.skipTest("No MongoDB instance running")
super(QueueLifecycleMongoDBTests, self).setUp()
self.cfg = config.namespace("drivers:storage:mongodb").from_options()
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from marconi.common import config
from marconi import transport
cfg = config.namespace("drivers:transport:wsgi").from_options(port=8888)
class Driver(transport.DriverBase):
def __init__(self, queue_controller, message_controller, claim_controller):
queues = transport.wsgi.QueuesResource(queue_controller)
self.app = api = falcon.API()
api.add_route("/v1/{tenant_id}/queues/{queue_name}", queues)
def listen(self):
pass
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from marconi.common import config
from marconi.queues.storage import base
from marconi.queues.storage import exceptions
from marconi.queues.storage.sqlite import utils
CFG = config.namespace('limits:storage').from_options(
default_message_paging=10,
)
class ClaimController(base.ClaimBase):
def __init__(self, driver):
self.driver = driver
self.driver.run('''
create table
if not exists
Claims (
id INTEGER PRIMARY KEY AUTOINCREMENT,
qid INTEGER,
ttl INTEGER,
created DATETIME, -- seconds since the Julian day
FOREIGN KEY(qid) references Queues(id) on delete cascade
def setUp(self):
if not os.environ.get('MONGODB_TEST_LIVE'):
self.skipTest('No MongoDB instance running')
super(ClaimsMongoDBTests, self).setUp()
self.cfg = config.namespace('drivers:storage:mongodb').from_options()
def test_wrong_type(self):
ns = config.namespace('local')
with testing.expect(config.cfg.Error):
ns.from_options(opt={})
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from stevedore import driver
from marconi.common import config
from marconi.common import decorators
from marconi.common import exceptions
from marconi.openstack.common import log
from marconi import transport # NOQA
PROJECT_CFG = config.project('marconi')
CFG = config.namespace('drivers').from_options(
transport='wsgi',
storage='sqlite')
LOG = log.getLogger(__name__)
class Bootstrap(object):
"""Defines the Marconi bootstrapper.
The bootstrap loads up drivers per a given configuration, and
manages their lifetimes.
"""
def __init__(self, config_file=None, cli_args=None):
PROJECT_CFG.load(filename=config_file, args=cli_args)
log.setup('marconi')
def setUp(self):
super(QueueLifecycleBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
def test_wrong_type(self):
ns = config.namespace('local')
with testtools.ExpectedException(config.cfg.Error):
ns.from_options(opt={})
# limitations under the License.
import falcon
from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils
LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
content_max_length=256 * 1024
)
MESSAGE_POST_SPEC = (('ttl', int), ('body', '*'))
class CollectionResource(object):
__slots__ = ('message_controller')
def __init__(self, message_controller):
self.message_controller = message_controller
#-----------------------------------------------------------------------
# Helpers
#-----------------------------------------------------------------------
field names will be, most of the time, the first
letter of their long name.
"""
import pymongo.errors
from marconi.common import config
import marconi.openstack.common.log as logging
from marconi.openstack.common import timeutils
from marconi import storage
from marconi.storage import exceptions
from marconi.storage.mongodb import utils
LOG = logging.getLogger(__name__)
CFG = config.namespace('limits:storage').from_options(
default_queue_paging=10,
)
class QueueController(storage.QueueBase):
"""Implements queue resource operations using MongoDB.
Queues:
Name Field
------------------
name -> n
project -> p
counter -> c
metadata -> m
"""
'database': 'marconi',
# Maximum number of times to retry a failed operation. Currently
# only used for retrying a message post.
'max_attempts': 1000,
# Maximum sleep interval between retries (actual sleep time
# increases linearly according to number of attempts performed).
'max_retry_sleep': 0.1,
# Maximum jitter interval, to be added to the sleep interval, in
# order to decrease probability that parallel requests will retry
# at the same instant.
'max_retry_jitter': 0.005,
# Frequency of message garbage collections, in seconds
'gc_interval': 5 * 60,
# Threshold of number of expired messages to reach in a given
# queue, before performing the GC. Useful for reducing frequent
# locks on the DB for non-busy queues, or for worker queues
# which process jobs quickly enough to keep the number of in-
# flight messages low.
#
# Note: The higher this number, the larger the memory-mapped DB
# files will be.
'gc_threshold': 1000,
}
CFG = config.namespace('drivers:storage:mongodb').from_options(**OPTIONS)
import falcon
from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils
LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
metadata_max_length=64 * 1024
)
class Resource(object):
__slots__ = ('queue_ctrl', )
def __init__(self, queue_controller):
self.queue_ctrl = queue_controller
def on_get(self, req, resp, project_id, queue_name):
LOG.debug(_(u'Queue metadata GET - queue: %(queue)s, '
u'project: %(project)s') %
{'queue': queue_name, 'project': project_id})
try:
def test_wrong_type(self):
ns = config.namespace('local')
with testtools.ExpectedException(config.cfg.Error):
ns.from_options(opt={})
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import time
from marconi.common import config
from marconi import storage
from marconi.storage import mongodb
from marconi.storage.mongodb import controllers
from marconi.tests.storage import base
from marconi.tests import util as testing
cfg = config.namespace("drivers:storage:mongodb").from_options()
class MongodbDriverTest(testing.TestBase):
def setUp(self):
if not os.environ.get("MONGODB_TEST_LIVE"):
self.skipTest("No MongoDB instance running")
super(MongodbDriverTest, self).setUp()
self.load_conf("wsgi_mongodb.conf")
def test_db_instance(self):
driver = mongodb.Driver()
db = driver.db
self.assertEquals(db.name, cfg.database)
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from marconi.common import config
from marconi.common import exceptions
from marconi.openstack.common import importutils
cfg_handle = config.project('marconi')
cfg = config.namespace('drivers').from_options(
transport='marconi.transport.wsgi',
storage='marconi.storage.sqlite')
class Bootstrap(object):
"""
Defines the Marconi Bootstrap
The bootstrap loads up drivers per a given configuration, and manages their
lifetimes.
"""
def __init__(self, config_file=None, cli_args=None):
cfg_handle.load(filename=config_file, args=cli_args)
self.storage_module = import_driver(cfg.storage)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import contextlib
import sqlite3
import msgpack
from marconi.common import config
from marconi import storage
from marconi.storage.sqlite import controllers
from marconi.storage.sqlite import utils
CFG = config.namespace('drivers:storage:sqlite').from_options(
database=':memory:')
class Driver(storage.DriverBase):
def __init__(self):
self.__path = CFG.database
self.__conn = sqlite3.connect(self.__path,
detect_types=sqlite3.PARSE_DECLTYPES)
self.__db = self.__conn.cursor()
self.run('''PRAGMA foreign_keys = ON''')
@staticmethod
def pack(o):
"""Converts a Python variable to a custom SQlite `DOCUMENT`.
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import sqlite3
from marconi.common import config
from marconi import storage
from marconi.storage import base
from marconi.storage import exceptions
cfg = config.namespace('drivers:storage:reference').from_options(
database=':memory:')
class Driver(storage.DriverBase):
def __init__(self):
self.__path = cfg.database
self.__conn = sqlite3.connect(self.__path)
self.__db = self.__conn.cursor()
def run(self, sql, *args):
return self.__db.execute(sql, args)
def get(self, sql, *args):
return self.run(sql, *args).fetchone()
def __enter__(self):
def setUp(self):
super(QueueLifecycleBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils
LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
content_max_length=256 * 1024)
MESSAGE_POST_SPEC = (('ttl', int), ('body', '*'))
class CollectionResource(object):
__slots__ = ('message_controller')
def __init__(self, message_controller):
self.message_controller = message_controller
#-----------------------------------------------------------------------
# Helpers
#-----------------------------------------------------------------------
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from marconi.common import config
from marconi import transport
cfg = config.namespace('drivers:transport:wsgi').from_options(port=8888)
class Driver(transport.DriverBase):
def __init__(self, queue_controller, message_controller, claim_controller):
queues = transport.wsgi.QueuesResource(queue_controller)
self.app = api = falcon.API()
api.add_route('/v1/{tenant_id}/queues/{queue_name}', queues)
def listen(self):
pass
import marconi.openstack.common.log as logging
from marconi import transport
from marconi.transport import auth
from marconi.transport.wsgi import claims
from marconi.transport.wsgi import messages
from marconi.transport.wsgi import queues
from marconi.transport.wsgi import stats
OPTIONS = {
'bind': '0.0.0.0',
'port': 8888
}
PROJECT_CFG = config.project('marconi')
GLOBAL_CFG = PROJECT_CFG.from_options()
WSGI_CFG = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS)
LOG = logging.getLogger(__name__)
def _extract_project_id(req, resp, params):
params['project_id'] = req.get_header('X-PROJECT-ID')
class Driver(transport.DriverBase):
def __init__(self, storage):
super(Driver, self).__init__(storage)
self.app = falcon.API(before=_extract_project_id)
def setUp(self):
if not os.environ.get('MONGODB_TEST_LIVE'):
self.skipTest('No MongoDB instance running')
super(ClaimsMongoDBTests, self).setUp()
self.cfg = config.namespace('drivers:storage:mongodb').from_options()
'database': 'marconi',
# Maximum number of times to retry a failed operation. Currently
# only used for retrying a message post.
'max_attempts': 1000,
# Maximum sleep interval between retries (actual sleep time
# increases linearly according to number of attempts performed).
'max_retry_sleep': 0.1,
# Maximum jitter interval, to be added to the sleep interval, in
# order to decrease probability that parallel requests will retry
# at the same instant.
'max_retry_jitter': 0.005,
# Frequency of message garbage collections, in seconds
'gc_interval': 5 * 60,
# Threshold of number of expired messages to reach in a given
# queue, before performing the GC. Useful for reducing frequent
# locks on the DB for non-busy queues, or for worker queues
# which process jobs quickly enough to keep the number of in-
# flight messages low.
#
# Note: The higher this number, the larger the memory-mapped DB
# files will be.
'gc_threshold': 1000,
}
CFG = config.namespace('drivers:storage:mongodb').from_options(**OPTIONS)
def test_wrong_type(self):
ns = config.namespace('local')
with testing.expect(config.cfg.Error):
ns.from_options(opt={})
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import contextlib
import sqlite3
import msgpack
from marconi.common import config
from marconi import storage
from marconi.storage.sqlite import controllers
from marconi.storage.sqlite import utils
CFG = config.namespace('drivers:storage:sqlite').from_options(
database=':memory:')
class Driver(storage.DriverBase):
def __init__(self):
self.__path = CFG.database
self.__conn = sqlite3.connect(self.__path,
detect_types=sqlite3.PARSE_DECLTYPES)
self.__db = self.__conn.cursor()
self.run('''PRAGMA foreign_keys = ON''')
@staticmethod
def pack(o):
"""Converts a Python variable to a custom SQlite `DOCUMENT`.
:param o: a Python str, unicode, int, long, float, bool, None
import simplejson as json
from marconi.common import config
from marconi.common import exceptions
OPTIONS = {
'queue_paging_uplimit': 20,
'metadata_size_uplimit': 64 * 1024,
'message_paging_uplimit': 20,
'message_size_uplimit': 256 * 1024,
'message_ttl_max': 1209600,
'claim_ttl_max': 43200,
'claim_grace_max': 43200,
}
CFG = config.namespace('limits:transport').from_options(**OPTIONS)
QUEUE_NAME_REGEX = re.compile('^[\w-]+$')
def queue_creation(name):
"""Restrictions on a queue name.
:param name: The queue name
:raises: ValidationFailed if the name is longer than 64 bytes or
contains bytes other than ASCII digits, letters, underscore,
and dash.
"""
if len(name) > 64:
raise exceptions.ValidationFailed(
Mongodb storage driver implementation
"""
import pymongo
import pymongo.errors
from marconi.common import config
from marconi import storage
from marconi.storage.mongodb import controllers
options = {
"uri": None,
"database": "marconi",
}
cfg = config.namespace('drivers:storage:mongodb').from_options(**options)
class Driver(storage.DriverBase):
def __init__(self):
self._database = None
@property
def db(self):
"""
Property for lazy instantiation of
mongodb's database.
"""
if not self._database:
if cfg.uri and 'replicaSet' in cfg.uri:
