class Parser(anyconfig.backend.base.StringParser,
anyconfig.backend.base.BinaryFilesMixin):
"""
Loader/Dumper of BSON files.
"""
_type = "bson"
_extensions = ["bson", "bsn"] # Temporary.
_load_opts = [] if bson.has_c() else ["codec_options"]
_dump_opts = [] if bson.has_c() else ["check_keys", "codec_options"]
_ordered = not bson.has_c()
_dict_opts = [] if bson.has_c() else ["document_class"]
def _load_options(self, container, **options):
"""
:param container: callble to make a container object later
"""
if "codec_options" not in options:
options.setdefault("document_class", container)
if any(k in options for k in _CO_OPTIONS):
options["codec_options"] = _codec_options(**options)
return anyconfig.utils.filter_options(self._load_opts, options)
def load_from_string(self, content, container, **kwargs):
"""
Load BSON config from given string `content`.
:param content: BSON config content in bytes data string
:param container: callble to make a container object
:param kwargs: optional keyword parameters
:return: Dict-like object holding config parameters
"""
if self._load_opts: # indicates that C extension is not used.
objs = bson.decode_all(content, **kwargs)
else:
# .. note::
# The order of loaded configuration keys may be lost but
# there is no way to avoid that, AFAIK.
objs = [container(x) for x in bson.decode_all(content)
if x is not None]
return objs[0] if objs else None
def dump_to_string(self, cnf, **kwargs):
"""Dump BSON data `cnf` to a string.
:param cnf: BSON Data to dump
:param kwargs: optional keyword parameters to be sanitized
:return: string represents the configuration
"""
if self._dump_opts:
container = self._container_factory(**kwargs)
opts = self._load_options(container, **kwargs)
for key in self._dump_opts:
if kwargs.get(key, False):
opts[key] = kwargs[key]
return bson.BSON.encode(cnf, *opts)
return bson.BSON.encode(cnf)
class Parser(anyconfig.backend.base.FromStringLoader,
anyconfig.backend.base.ToStringDumper):
"""
Loader/Dumper of BSON files.
"""
_type = "bson"
_extensions = ["bson", "bsn"] # Temporary.
_load_opts = [] if bson.has_c() else ["tz_aware", "uuid_subtype"]
_dump_opts = ["check_keys", "uuid_subtype"]
_open_flags = ('rb', 'wb')
dump_to_string = anyconfig.backend.base.to_method(bson.BSON.encode)
def load_from_string(self, content, to_container, **kwargs):
"""
Load BSON config from given string `content`.
:param content: BSON config content in bytes data string
:param to_container: callble to make a container object
:param kwargs: optional keyword parameters
:return: Dict-like object holding config parameters
"""
if self._load_opts: # indicates that C extension is not used.
objs = bson.decode_all(content, as_class=to_container, **kwargs)
else:
# .. note::
# The order of loaded configuration keys may be lost but
# there is no way to avoid that, AFAIK.
objs = to_container(bson.decode_all(content))
return objs[0] if objs else None
class Test_10(TBC.Test_10_dumps_and_loads, HasParserTrait):
# Can't if bson.has_c():
if not bson.has_c():
load_options = dict(as_class=dict)
dump_options = dict(check_keys=True)
empty_patterns = ['']
def init(fast_db_url, big_db_url):
"""Initialize FDB/BDB factory singletons.
As a useful side effect, creates all (missing) indices.
@type fast_db_url: basestring
@type big_db_url: basestring
@raises MongoDBInitializationException: if cannot complete connection.
"""
global FDB, BDB
assert FDB is None and BDB is None, (FDB, BDB)
try:
FDB = DocStoreWrapperFactory(fast_db_url)
BDB = GFSEnabledDocStoreWrapperFactory(big_db_url)
except MongoDBInitializationException:
raise
if not bson.has_c():
logger.warning('python-bson-ext is not installed, '
'performance may be lower than expected!')
if not pymongo.has_c():
logger.warning('python-pymongo-ext is not installed, '
'performance may be lower than expected!')
logger.debug('Creating indices on Fast DataBase')
with FDB() as fdbw:
__make_indices(fdbqueries.FDB_INDICES_PER_COLLECTION.iteritems(),
dsw=fdbw)
logger.debug('Creating indices on Big DataBase')
with BDB() as bdbw:
__make_indices(bdbqueries.GFS_INDICES_PER_COLLECTION.iteritems(),
dsw=bdbw)
def init(fast_db_url, big_db_url):
"""Initialize FDB/BDB factory singletons.
As a useful side effect, creates all (missing) indices.
@type fast_db_url: basestring
@type big_db_url: basestring
@raises MongoDBInitializationException: if cannot complete connection.
"""
global FDB, BDB
assert FDB is None and BDB is None, (FDB, BDB)
try:
FDB = DocStoreWrapperFactory(fast_db_url)
BDB = GFSEnabledDocStoreWrapperFactory(big_db_url)
except MongoDBInitializationException:
raise
if not bson.has_c():
logger.warning('python-bson-ext is not installed, '
'performance may be lower than expected!')
if not pymongo.has_c():
logger.warning('python-pymongo-ext is not installed, '
'performance may be lower than expected!')
logger.debug('Creating indices on Fast DataBase')
with FDB() as fdbw:
__make_indices(fdbqueries.FDB_INDICES_PER_COLLECTION.iteritems(),
dsw=fdbw)
logger.debug('Creating indices on Big DataBase')
with BDB() as bdbw:
__make_indices(bdbqueries.GFS_INDICES_PER_COLLECTION.iteritems(),
dsw=bdbw)
def setup_app():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
if settings.conf.debug and settings.conf.ssl:
settings.conf.ssl = False
def setup_app():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
if settings.conf.debug and settings.conf.ssl:
settings.conf.ssl = False
def test_dates(self):
doc = {"early": datetime.datetime(1686, 5, 5), "late": datetime.datetime(2086, 5, 5)}
try:
self.assertEqual(doc, BSON.encode(doc).decode())
except ValueError:
# Ignore ValueError when no C ext, since it's probably
# a problem w/ 32-bit Python - we work around this in the
# C ext, though.
if bson.has_c():
raise
def test_dates(self):
doc = {"early": datetime.datetime(1686, 5, 5),
"late": datetime.datetime(2086, 5, 5)}
try:
self.assertEqual(doc, BSON.encode(doc).decode())
except ValueError:
# Ignore ValueError when no C ext, since it's probably
# a problem w/ 32-bit Python - we work around this in the
# C ext, though.
if bson.has_c():
raise
# Copyright 2009-2014 MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fail if the C extension module doesn't exist.
Only really intended to be used by internal build scripts.
"""
import sys
sys.path[0:0] = [""]
import bson
import pymongo
if not pymongo.has_c() or not bson.has_c():
sys.exit("could not load C extensions")
def setup_check():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
def setup_check():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
def setup_mongo():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
prefix = settings.conf.mongodb_collection_prefix or ''
last_error = time.time() - 24
while True:
try:
client = pymongo.MongoClient(settings.conf.mongodb_url,
connectTimeoutMS=2000)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception('Error connecting to mongodb server')
database = client.get_default_database()
cur_collections = database.collection_names()
if prefix + 'messages' not in cur_collections:
database.create_collection(prefix + 'messages',
capped=True,
size=100000)
mongo.collections.update({
'transaction':
getattr(database, prefix + 'transaction'),
'queue':
getattr(database, prefix + 'queue'),
'task':
getattr(database, prefix + 'task'),
'system':
getattr(database, prefix + 'system'),
'messages':
getattr(database, prefix + 'messages'),
'administrators':
getattr(database, prefix + 'administrators'),
'users':
getattr(database, prefix + 'users'),
'users_key_link':
getattr(database, prefix + 'users_key_link'),
'organizations':
getattr(database, prefix + 'organizations'),
'hosts':
getattr(database, prefix + 'hosts'),
'hosts_usage':
getattr(database, prefix + 'hosts_usage'),
'servers':
getattr(database, prefix + 'servers'),
'servers_output':
getattr(database, prefix + 'servers_output'),
'servers_bandwidth':
getattr(database, prefix + 'servers_bandwidth'),
'servers_ip_pool':
getattr(database, prefix + 'servers_ip_pool'),
'dh_params':
getattr(database, prefix + 'dh_params'),
'auth_nonces':
getattr(database, prefix + 'auth_nonces'),
'auth_limiter':
getattr(database, prefix + 'auth_limiter'),
'otp':
getattr(database, prefix + 'otp'),
'otp_cache':
getattr(database, prefix + 'otp_cache'),
})
if prefix + 'log_entries' not in cur_collections:
log_limit = settings.app.log_entry_limit
database.create_collection(prefix + 'log_entries',
capped=True,
size=log_limit * 256 * 2,
max=log_limit)
mongo.collections.update({
'log_entries':
getattr(database, prefix + 'log_entries'),
})
for collection_name, collection in mongo.collections.items():
collection.name_str = collection_name
settings.init()
mongo.collections['transaction'].ensure_index('lock_id', unique=True)
mongo.collections['transaction'].ensure_index([
('ttl_timestamp', pymongo.ASCENDING),
('state', pymongo.ASCENDING),
('priority', pymongo.DESCENDING),
])
mongo.collections['queue'].ensure_index('runner_id')
mongo.collections['queue'].ensure_index('ttl_timestamp')
mongo.collections['task'].ensure_index('type', unique=True)
mongo.collections['task'].ensure_index('ttl_timestamp')
mongo.collections['log_entries'].ensure_index([
('timestamp', pymongo.DESCENDING),
])
mongo.collections['messages'].ensure_index('channel')
mongo.collections['administrators'].ensure_index('username', unique=True)
mongo.collections['users'].ensure_index([
('type', pymongo.ASCENDING),
('org_id', pymongo.ASCENDING),
])
mongo.collections['users'].ensure_index([
('org_id', pymongo.ASCENDING),
('name', pymongo.ASCENDING),
])
mongo.collections['users_key_link'].ensure_index('key_id')
mongo.collections['users_key_link'].ensure_index('short_id', unique=True)
mongo.collections['organizations'].ensure_index('type')
mongo.collections['hosts'].ensure_index('name')
mongo.collections['hosts_usage'].ensure_index([
('host_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers'].ensure_index('name')
mongo.collections['servers'].ensure_index('ping_timestamp')
mongo.collections['servers_output'].ensure_index([
('server_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers_bandwidth'].ensure_index([
('server_id', pymongo.ASCENDING),
('period', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers_ip_pool'].ensure_index([
('server_id', pymongo.ASCENDING),
('user_id', pymongo.ASCENDING),
])
mongo.collections['servers_ip_pool'].ensure_index('user_id')
mongo.collections['dh_params'].ensure_index('dh_param_bits')
mongo.collections['auth_nonces'].ensure_index([
('token', pymongo.ASCENDING),
('nonce', pymongo.ASCENDING),
],
unique=True)
mongo.collections['users_key_link'].ensure_index(
'timestamp', expireAfterSeconds=settings.app.key_link_timeout)
mongo.collections['auth_nonces'].ensure_index(
'timestamp', expireAfterSeconds=settings.app.auth_time_window * 2.1)
mongo.collections['auth_limiter'].ensure_index(
'timestamp', expireAfterSeconds=settings.app.auth_limiter_ttl)
mongo.collections['otp'].ensure_index('timestamp', expireAfterSeconds=120)
mongo.collections['otp_cache'].ensure_index(
'timestamp', expireAfterSeconds=settings.user.otp_cache_ttl)
if not auth.Administrator.collection.find_one():
auth.Administrator(
username=DEFAULT_USERNAME,
password=DEFAULT_PASSWORD,
default=True,
).commit()
secret_key = settings.app.cookie_secret
if not secret_key:
secret_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.cookie_secret = secret_key
settings.commit()
app.app.secret_key = secret_key.encode()
server_api_key = settings.app.server_api_key
if not server_api_key:
server_api_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.server_api_key = server_api_key
settings.commit()
def setup_mongo():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
prefix = settings.conf.mongodb_collection_prefix or ''
last_error = time.time() - 24
while True:
try:
client = pymongo.MongoClient(settings.conf.mongodb_url,
connectTimeoutMS=2000)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception('Error connecting to mongodb server')
database = client.get_default_database()
cur_collections = database.collection_names()
if prefix + 'messages' not in cur_collections:
database.create_collection(prefix + 'messages', capped=True,
size=100000)
mongo.collections.update({
'transaction': getattr(database, prefix + 'transaction'),
'queue': getattr(database, prefix + 'queue'),
'task': getattr(database, prefix + 'task'),
'system': getattr(database, prefix + 'system'),
'messages': getattr(database, prefix + 'messages'),
'administrators': getattr(database, prefix + 'administrators'),
'users': getattr(database, prefix + 'users'),
'users_key_link': getattr(database, prefix + 'users_key_link'),
'organizations': getattr(database, prefix + 'organizations'),
'hosts': getattr(database, prefix + 'hosts'),
'hosts_usage': getattr(database, prefix + 'hosts_usage'),
'servers': getattr(database, prefix + 'servers'),
'servers_output': getattr(database, prefix + 'servers_output'),
'servers_bandwidth': getattr(database, prefix + 'servers_bandwidth'),
'servers_ip_pool': getattr(database, prefix + 'servers_ip_pool'),
'dh_params': getattr(database, prefix + 'dh_params'),
'auth_nonces': getattr(database, prefix + 'auth_nonces'),
'auth_limiter': getattr(database, prefix + 'auth_limiter'),
'otp': getattr(database, prefix + 'otp'),
'otp_cache': getattr(database, prefix + 'otp_cache'),
})
if prefix + 'log_entries' not in cur_collections:
log_limit = settings.app.log_entry_limit
database.create_collection(prefix + 'log_entries', capped=True,
size=log_limit * 256 * 2, max=log_limit)
mongo.collections.update({
'log_entries': getattr(database, prefix + 'log_entries'),
})
for collection_name, collection in mongo.collections.items():
collection.name_str = collection_name
settings.init()
mongo.collections['transaction'].ensure_index('lock_id', unique=True)
mongo.collections['transaction'].ensure_index([
('ttl_timestamp', pymongo.ASCENDING),
('state', pymongo.ASCENDING),
('priority', pymongo.DESCENDING),
])
mongo.collections['queue'].ensure_index('runner_id')
mongo.collections['queue'].ensure_index('ttl_timestamp')
mongo.collections['task'].ensure_index('type', unique=True)
mongo.collections['task'].ensure_index('ttl_timestamp')
mongo.collections['log_entries'].ensure_index([
('timestamp', pymongo.DESCENDING),
])
mongo.collections['messages'].ensure_index('channel')
mongo.collections['administrators'].ensure_index('username', unique=True)
mongo.collections['users'].ensure_index([
('type', pymongo.ASCENDING),
('org_id', pymongo.ASCENDING),
])
mongo.collections['users'].ensure_index([
('org_id', pymongo.ASCENDING),
('name', pymongo.ASCENDING),
])
mongo.collections['users_key_link'].ensure_index('key_id')
mongo.collections['users_key_link'].ensure_index('short_id', unique=True)
mongo.collections['organizations'].ensure_index('type')
mongo.collections['hosts'].ensure_index('name')
mongo.collections['hosts_usage'].ensure_index([
('host_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers'].ensure_index('name')
mongo.collections['servers'].ensure_index('ping_timestamp')
mongo.collections['servers_output'].ensure_index([
('server_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers_bandwidth'].ensure_index([
('server_id', pymongo.ASCENDING),
('period', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers_ip_pool'].ensure_index([
('server_id', pymongo.ASCENDING),
('user_id', pymongo.ASCENDING),
])
mongo.collections['servers_ip_pool'].ensure_index('user_id')
mongo.collections['dh_params'].ensure_index('dh_param_bits')
mongo.collections['auth_nonces'].ensure_index([
('token', pymongo.ASCENDING),
('nonce', pymongo.ASCENDING),
], unique=True)
mongo.collections['users_key_link'].ensure_index('timestamp',
expireAfterSeconds=settings.app.key_link_timeout)
mongo.collections['auth_nonces'].ensure_index('timestamp',
expireAfterSeconds=settings.app.auth_time_window * 2.1)
mongo.collections['auth_limiter'].ensure_index('timestamp',
expireAfterSeconds=settings.app.auth_limiter_ttl)
mongo.collections['otp'].ensure_index('timestamp',
expireAfterSeconds=120)
mongo.collections['otp_cache'].ensure_index('timestamp',
expireAfterSeconds=settings.user.otp_cache_ttl)
if not auth.Administrator.collection.find_one():
auth.Administrator(
username=DEFAULT_USERNAME,
password=DEFAULT_PASSWORD,
default=True,
).commit()
secret_key = settings.app.cookie_secret
if not secret_key:
secret_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.cookie_secret = secret_key
settings.commit()
app.app.secret_key = secret_key.encode()
server_api_key = settings.app.server_api_key
if not server_api_key:
server_api_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.server_api_key = server_api_key
settings.commit()
if module in needs_replacing:
del sys.modules[name]
for module, attr in ((bson, '_cbson'), (pymongo, '_cmessage'),
(pymongo.message, '_cmessage')):
try:
delattr(module, attr)
except AttributeError:
pass
del attr, name, module, needs_replacing
importlib.reload(bson)
importlib.reload(pymongo.message)
assert not bson.has_c()
assert not pymongo.has_c()
import types, functools
import bson, bson.son, contextlib, struct
from bson.binary import Binary, OLD_UUID_SUBTYPE
from gridfs import GridFS
from collections import OrderedDict
from bson import _elements_to_dict, DBRef
from pytransact import spickle
max_bson_element_size = 2 ** 18
Extension = spickle.ExtensionType()
