def _open(self):
conninfo = self.connection.client
mongoconn = Connection(host=conninfo.hostname, port=conninfo.port)
dbname = conninfo.virtual_host
version = mongoconn.server_info()["version"]
if tuple(map(int, version.split(".")[:2])) < (1, 3):
raise NotImplementedError(
"Kombu requires MongoDB version 1.3+, but connected to %s" % (
version, ))
if not dbname or dbname == "/":
dbname = "kombu_default"
database = getattr(mongoconn, dbname)
if conninfo.userid:
database.authenticate(conninfo.userid, conninfo.password)
self.db = database
col = database.messages
col.ensure_index([("queue", 1)])
if "messages.broadcast" not in database.collection_names():
capsize = conninfo.transport_options.get(
"capped_queue_size") or 100000
database.create_collection("messages.broadcast", size=capsize,
capped=True)
self.bcast = getattr(database, "messages.broadcast")
self.bcast.ensure_index([("queue", 1)])
self.routing = getattr(database, "messages.routing")
self.routing.ensure_index([("queue", 1), ("exchange", 1)])
return database
def test_document_class(self):
c = Connection(self.host, self.port)
db = c.pymongo_test
db.test.insert({"x": 1})
self.assertEqual(dict, c.document_class)
self.assertTrue(isinstance(db.test.find_one(), dict))
self.assertFalse(isinstance(db.test.find_one(), SON))
c.document_class = SON
self.assertEqual(SON, c.document_class)
self.assertTrue(isinstance(db.test.find_one(), SON))
self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON))
c = Connection(self.host, self.port, document_class=SON)
db = c.pymongo_test
self.assertEqual(SON, c.document_class)
self.assertTrue(isinstance(db.test.find_one(), SON))
self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON))
c.document_class = dict
self.assertEqual(dict, c.document_class)
self.assertTrue(isinstance(db.test.find_one(), dict))
self.assertFalse(isinstance(db.test.find_one(), SON))
class TestConfig(unittest.TestCase):
def setUp(self):
filename = dirname(__file__) + '/logging-test.config'
fileConfig(filename)
""" Create an empty database that could be used for logging """
self.db_name = '_mongolog_test'
self.collection_name = 'log_test'
self.conn = Connection('localhost')
self.conn.drop_database(self.db_name)
def tearDown(self):
""" Drop used database """
self.conn.drop_database(self.db_name)
def testLoggingFileConfiguration(self):
log = logging.getLogger('example')
log.debug('test')
r = self.conn[self.db_name][self.collection_name]
message = r.find_one({'level': 'debug', 'msg': 'test'})
self.assertEquals(message['msg'], 'test')
def __init__(self):
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, 'config', 'processing.conf'))
self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled")
self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled")
self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled")
if self.__mongodbEnabled:
#Anbindung an Datenbank MongoDB Collection Ragpicker herstellen
try:
mongodbHost = self.__cfgReporting.getOption("mongodb", "host")
mongodbPort = self.__cfgReporting.getOption("mongodb", "port")
self.__mongodbConnection = Connection(mongodbHost, mongodbPort)
self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker
self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families
self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue
except TypeError:
raise Exception("MongoDB connection port in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (ragpicker)")
if self.__codedbEnabled:
#Anbindung an Datenbank MongoDB Collection CodeDB herstellen
try:
codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host")
codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port")
self.__codedbConnection = Connection(codedbHost, codedbPort)
self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB
except TypeError:
raise Exception("MongoDB connection port for CodeDB in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (codeDB)")
def test_ipv6(self):
self.assertRaises(InvalidURI, _parse_uri, "::1", 27017)
self.assertRaises(InvalidURI, _parse_uri, "[::1", 27017)
self.assertRaises(InvalidURI, _parse_uri, "::1]:27017")
self.assertRaises(InvalidURI, _parse_uri, "mongodb://::1", 27017)
self.assert_(_parse_uri, "mongodb://[::1]:27017/?slaveOk=true")
self.assert_(_parse_uri,
"[::1]:27017,[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"
":27018,192.168.0.212:27019,localhost:27020")
self.assert_(_parse_uri,
"mongodb://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"
":27017/?slaveOk=true")
try:
connection = Connection("[::1]")
except:
# Either mongod was started without --ipv6
# or the OS doesn't support it (or both).
raise SkipTest()
# Try a few simple things
connection = Connection("mongodb://[::1]:27017")
connection = Connection("mongodb://[::1]:27017/?slaveOk=true")
connection = Connection("[::1]:27017,localhost:27017")
connection = Connection("localhost:27017,[::1]:27017")
connection.pymongo_test.test.save({"dummy": u"object"})
connection.pymongo_test_bernie.test.save({"dummy": u"object"})
dbs = connection.database_names()
self.assert_("pymongo_test" in dbs)
self.assert_("pymongo_test_bernie" in dbs)
class plantmongo:
def __init__(self, addr, user="******", password="******"):
self.addr = addr
self.user = user
self.password = password
self.conn = Connection(addr)
self.db = conn.plant
if not db.authenticate("plant", "plant"):
print "unable to authenticate to mongodb"
self.connected = False
return None
self.coll = self.db.test
def publish(self, xbeeobj):
post = {
"time": time.time(),
"temp": xbeeobj.temp,
"moisture": xbeeobj.moisture,
"dewpoint": xbeeobj.dewpoint,
"pressure": xbeeobj.pressure,
"light": xbeeobj.light,
"batt": xbeeobj.batt,
"rssi": xbeeobj.rssi,
}
self.coll.save(post)
def close(self):
self.conn.disconnect()
def test_connection(self):
c = Connection(host, port)
self.assertTrue(c.auto_start_request)
self.assertFalse(c.slave_okay)
self.assertFalse(c.safe)
self.assertEqual({}, c.get_lasterror_options())
# Connection's writes are unacknowledged by default
doc = {"_id": ObjectId()}
coll = c.pymongo_test.write_concern_test
coll.really_drop()
coll.insert(doc)
coll.insert(doc)
c = Connection("mongodb://%s:%s/?safe=true" % (host, port))
self.assertTrue(c.safe)
# Connection's network_timeout argument is translated into
# socketTimeoutMS
self.assertEqual(123, Connection(
host, port, network_timeout=123)._MongoClient__net_timeout)
for network_timeout in 'foo', 0, -1:
self.assertRaises(
ConfigurationError,
Connection, host, port, network_timeout=network_timeout)
def test_unix_domain_socket(self):
try:
connection = Connection("/tmp/mongodb-27017.sock")
connection.database_names()
except:
# Possibly the OS doesn't support unix domain sockets
raise SkipTest("No Unix domain sockets")
class MongoHandler(logging.Handler):
""" Custom log handler
Logs all messages to a mongo collection. This handler is
designed to be used with the standard python logging mechanism.
"""
@classmethod
def to(cls, db, collection, host='localhost', port=None, level=logging.NOTSET):
""" Create a handler for a given """
return cls(Connection(host, port)[db][collection], level)
def __init__(self, collection, db='mongolog', host='localhost', port=None, level=logging.NOTSET):
""" Init log handler and store the collection handle """
logging.Handler.__init__(self, level)
if (type(collection) == str):
self.collection = Connection(host, port)[db][collection]
else:
self.collection = collection
self.formatter = MongoFormatter()
def emit(self,record):
""" Store the record to the collection. Async insert """
try:
self.collection.save(self.format(record))
except InvalidDocument, e:
logging.error("Unable to save log record: %s", e.message ,exc_info=True)
def WriteTempFile(self, data, hash_name=None):
if self.use_cache == True:
if hash_name is None:
hash = md5(self.url )
hash_name = hash.hexdigest()
self.last_hash_name = hash_name
self.log.debug('write file to cache: ', hash_name)
self.log.debug('use mongo: %s' % self.use_mongo)
# open(self.download_temp+hash_name, 'wb').write(data)
if self.use_mongo == False:
f_name = self.download_temp + hash_name + '.gz'
f = gzip.open(f_name, 'wb')
f.write(data)
f.close()
if self.use_mongo == True:
connection = Connection("localhost", 27017)
db = connection['parser']
s = StringIO.StringIO()
f = gzip.GzipFile(fileobj=s, mode='wb')
f.write(data)
f.close()
val = s.getvalue()
s.close()
del (s)
del (f)
fs = GridFS(db)
fp = fs.open(hash_name , 'w', self.download_temp.replace('/', '') )
fp.write(val)
fp.close()
connection.disconnect()
def test_from_uri(self):
c = Connection(self.host, self.port)
self.assertRaises(InvalidURI, Connection.from_uri, "mongodb://localhost/baz")
self.assertEqual(c, Connection.from_uri("mongodb://%s:%s" %
(self.host, self.port)))
c.admin.system.users.remove({})
c.pymongo_test.system.users.remove({})
c.admin.add_user("admin", "pass")
c.pymongo_test.add_user("user", "pass")
self.assertRaises(InvalidURI, Connection.from_uri,
"mongodb://*****:*****@%s:%s" % (self.host, self.port))
self.assertRaises(InvalidURI, Connection.from_uri,
"mongodb://*****:*****@%s:%s" % (self.host, self.port))
self.assertRaises(InvalidURI, Connection.from_uri,
"mongodb://*****:*****@%s:%s" % (self.host, self.port))
Connection.from_uri("mongodb://*****:*****@%s:%s" % (self.host, self.port))
self.assertRaises(InvalidURI, Connection.from_uri,
"mongodb://*****:*****@%s:%s/pymongo_test" %
(self.host, self.port))
self.assertRaises(InvalidURI, Connection.from_uri,
"mongodb://*****:*****@%s:%s/pymongo_test" %
(self.host, self.port))
Connection.from_uri("mongodb://*****:*****@%s:%s/pymongo_test" %
(self.host, self.port))
def test_connection(self):
c = Connection(host, port)
self.assertTrue(c.auto_start_request)
self.assertEqual(None, c.max_pool_size)
self.assertFalse(c.slave_okay)
self.assertFalse(c.safe)
self.assertEqual({}, c.get_lasterror_options())
# Connection's writes are unacknowledged by default
doc = {"_id": ObjectId()}
coll = c.pymongo_test.write_concern_test
coll.drop()
coll.insert(doc)
coll.insert(doc)
c = Connection("mongodb://%s:%s/?safe=true" % (host, port))
self.assertTrue(c.safe)
# To preserve legacy Connection's behavior, max_size should be None.
# Pool should handle this without error.
self.assertEqual(None, get_pool(c).max_size)
c.end_request()
# Connection's network_timeout argument is translated into
# socketTimeoutMS
self.assertEqual(123, Connection(
host, port, network_timeout=123)._MongoClient__net_timeout)
for network_timeout in 'foo', 0, -1:
self.assertRaises(
ConfigurationError,
Connection, host, port, network_timeout=network_timeout)
def test_autoreconnect(self):
def find_one(conn):
return conn.test.stuff.find_one()
# Simulate a temporary connection failure
c = Connection('foo', _connect=False)
self.assertRaises(AutoReconnect, find_one, c)
c._Connection__nodes = set([('localhost', 27017)])
self.assert_(find_one, c)
def test_database_names(self):
connection = Connection(self.host, self.port)
connection.pymongo_test.test.save({"dummy": u"object"})
connection.pymongo_test_mike.test.save({"dummy": u"object"})
dbs = connection.database_names()
self.assertTrue("pymongo_test" in dbs)
self.assertTrue("pymongo_test_mike" in dbs)
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.das = DASCore(debug=debug)
config = deepcopy(das_readconfig())
dburi = config['mongodb']['dburi']
connection = Connection(dburi)
connection.drop_database('das')
def test_end_request(self):
self.skip()
connection = Connection([self.left, self.right])
db = connection.pymongo_test
for _ in range(100):
db.test.remove({})
db.test.insert({})
self.assertTrue(db.test.find_one())
connection.end_request()
def check_conn (ip = "localhost", port = 27017):
try:
port = int(port)
conn = Connection (host = ip, port = port)
except Exception as E:
logging.error (E)
return False
else:
conn.close ()
return True
def GetTempFile(self):
"""
Metoda pobiera/zapisuje stronke pobierana do cache'u
"""
data = None
if self.use_cache == True:
hash = md5(self.url)
self.hash_name = hash.hexdigest()
self.page_from_cache = False
# f_name = self.download_temp + self.hash_name
f_name = self.download_temp + self.hash_name + '.gz'
if self._devel == True:
print 'seek cache: ',f_name, '::', self.url
# czy plik lokalny jest gz
if os.path.exists(f_name.replace('.gz', '') ):
data = open(f_name.replace('.gz', ''), 'rb').read()
f = gzip.open(f_name, 'wb')
f.write(data)
f.close()
os.unlink( f_name.replace('.gz', '') )
return data
# teraz odczyt pliku gzip
if self.read_cache == True:
if self.use_mongo == True:
try:
connection = Connection("localhost", 27017)
db = connection['parser']
fs = GridFS(db)
fp = fs.open(self.hash_name , 'r', self.download_temp.replace('/', '') )
f = gzip.GzipFile(fileobj=fp, mode='rb')
data = f.read()
f.close()
fp.close()
del(f)
connection.disconnect()
except Exception, e:
print 'read cahce error: ', e
self.page_from_cache = False
return None
elif os.path.exists(f_name):
f = gzip.open(f_name, 'rb')
data = f.read()
f.close()
else:
data = ''
if self._devel == True:
print '# Found cache: ', self.hash_name
self.page_from_cache = True
def test_parse_uri(self):
self.assertEqual(([("localhost", 27017)], None, None, None),
Connection._parse_uri("localhost"))
self.assertRaises(InvalidURI, Connection._parse_uri, "http://foobar.com")
self.assertRaises(InvalidURI, Connection._parse_uri, "http://[email protected]")
self.assertEqual(([("localhost", 27017)], None, None, None),
Connection._parse_uri("mongodb://localhost"))
self.assertEqual(([("localhost", 27017)], None, "fred", "foobar"),
Connection._parse_uri("mongodb://*****:*****@localhost"))
self.assertEqual(([("localhost", 27017)], "baz", "fred", "foobar"),
Connection._parse_uri("mongodb://*****:*****@localhost/baz"))
self.assertEqual(([("example1.com", 27017), ("example2.com", 27017)],
None, None, None),
Connection._parse_uri("mongodb://example1.com:27017,example2.com:27017"))
self.assertEqual(([("localhost", 27017),
("localhost", 27018),
("localhost", 27019)], None, None, None),
Connection._parse_uri("mongodb://localhost,localhost:27018,localhost:27019"))
self.assertEqual(([("localhost", 27018)], None, None, None),
Connection._parse_uri("localhost:27018"))
self.assertEqual(([("localhost", 27017)], "foo", None, None),
Connection._parse_uri("localhost/foo"))
self.assertEqual(([("localhost", 27017)], None, None, None),
Connection._parse_uri("localhost/"))
def test_tz_aware(self):
aware = Connection(self.host, self.port, tz_aware=True)
naive = Connection(self.host, self.port)
aware.pymongo_test.drop_collection("test")
now = datetime.datetime.utcnow()
aware.pymongo_test.test.insert({"x": now}, safe=True)
self.assertEqual(None, naive.pymongo_test.test.find_one()["x"].tzinfo)
self.assertEqual(utc, aware.pymongo_test.test.find_one()["x"].tzinfo)
self.assertEqual(
aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None),
naive.pymongo_test.test.find_one()["x"])
def test_disconnect(self):
c = Connection(self.host, self.port)
coll = c.pymongo_test.bar
c.disconnect()
c.disconnect()
coll.count()
c.disconnect()
c.disconnect()
coll.count()
def __init__(self,
collection,
db='mongolog',
host='localhost',
port=None,
level=logging.NOTSET):
""" Init log handler and store the collection handle """
logging.Handler.__init__(self, level)
if (type(collection) == str):
self.collection = Connection(host, port)[db][collection]
else:
self.collection = collection
self.formatter = MongoFormatter()
def test_constants(self):
Connection.HOST = self.host
Connection.PORT = self.port
self.assert_(Connection())
Connection.HOST = "somedomainthatdoesntexist.org"
Connection.PORT = 123456789
self.assertRaises(ConnectionFailure, Connection)
self.assert_(Connection(self.host, self.port))
Connection.HOST = self.host
Connection.PORT = self.port
self.assert_(Connection())
def test_max_pool_size_validation(self):
self.assertRaises(ValueError, Connection, max_pool_size=-1)
self.assertRaises(TypeError, Connection, max_pool_size='foo')
self.assertRaises(TypeError, Connection,
'mongodb://localhost/?maxPoolSize=-1')
self.assertRaises(TypeError, Connection,
'mongodb://localhost/?maxPoolSize=foo')
self.assertRaises(TypeError, Connection,
'mongodb://localhost/?maxPoolSize=5.5')
c = Connection('mongodb://localhost/?maxPoolSize=5')
self.assertEqual(c.max_pool_size, 5)
c = Connection(max_pool_size=100)
self.assertEqual(c.max_pool_size, 100)
class MongoDBStore(SimpleStore):
'''MongoDB-based object storage frontend.'''
init = 'mongodb://'
def __init__(self, engine, **kw):
super(MongoDBStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, dbpath, self._colpath = spliturl.path.split('/')
self._conn = Connection(host=spliturl.hostname, port=spliturl.port)
self._db = getattr(self._conn, dbpath)
self._store = getattr(self._db, self._colpath)
self._store.ensure_index('key', unique=True)
def __getitem__(self, key):
try:
return self.loads(self._store.find_one(dict(key=key))['value'])
except TypeError:
raise KeyError(key)
def __setitem__(self, key, value):
self._store.save(dict(key=key, value=Binary(self.dumps(value))))
def __delitem__(self, key):
self._store.remove(dict(key=key))
def __len__(self):
return self._store.count()
def __iter__(self):
for key in self._store.find(
dict(key={'$exists': True}),
fields=['key'],
):
yield key['key']
def close(self):
self._conn.close()
def clear(self):
self._store.drop()
self._store = getattr(self._db, self._colpath)
def items(self):
for key in self._store.find({'key': {'$exists': True}}):
yield (key['key'], key['value'])
def values(self):
loads = self.loads
for value in self._store.distinct('key'):
yield loads(value['value'])
def _open(self):
"""
See mongodb uri documentation:
http://www.mongodb.org/display/DOCS/Connections
"""
client = self.connection.client
hostname = client.hostname or DEFAULT_HOST
authdb = dbname = client.virtual_host
if dbname in ["/", None]:
dbname = "kombu_default"
authdb = "admin"
if not client.userid:
hostname = hostname.replace('/' + client.virtual_host, '/')
else:
hostname = hostname.replace('/' + client.virtual_host,
'/' + authdb)
mongo_uri = 'mongodb://' + hostname
# At this point we expect the hostname to be something like
# (considering replica set form too):
#
# mongodb://[username:password@]host1[:port1][,host2[:port2],
# ...[,hostN[:portN]]][/[?options]]
mongoconn = Connection(host=mongo_uri, ssl=client.ssl)
database = getattr(mongoconn, dbname)
version = mongoconn.server_info()['version']
if tuple(map(int, version.split('.')[:2])) < (1, 3):
raise NotImplementedError(
'Kombu requires MongoDB version 1.3+ (server is {0})'.format(
version))
self.db = database
col = database.messages
col.ensure_index([('queue', 1), ('_id', 1)], background=True)
if 'messages.broadcast' not in database.collection_names():
capsize = (client.transport_options.get('capped_queue_size')
or 100000)
database.create_collection('messages.broadcast',
size=capsize,
capped=True)
self.bcast = getattr(database, 'messages.broadcast')
self.bcast.ensure_index([('queue', 1)])
self.routing = getattr(database, 'messages.routing')
self.routing.ensure_index([('queue', 1), ('exchange', 1)])
return database
class MongoDBStore(SimpleStore):
'''MongoDB-based object storage frontend.'''
init = 'mongodb://'
def __init__(self, engine, **kw):
super(MongoDBStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, dbpath, self._colpath = spliturl.path.split('/')
self._conn = Connection(host=spliturl.hostname, port=spliturl.port)
self._db = getattr(self._conn, dbpath)
self._store = getattr(self._db, self._colpath)
self._store.ensure_index('key', unique=True)
def __getitem__(self, key):
try:
return self.loads(self._store.find_one(dict(key=key))['value'], key)
except TypeError:
raise KeyError(key)
def __setitem__(self, key, value):
self._store.save(dict(key=key, value=Binary(self.dumps(value))))
def __delitem__(self, key):
self._store.remove(dict(key=key))
def __len__(self):
return self._store.count()
def __iter__(self):
for key in self._store.find(
dict(key={'$exists': True}), fields=['key'],
):
yield key['key']
def close(self):
self._conn.close()
def clear(self):
self._store.drop()
self._store = getattr(self._db, self._colpath)
def items(self):
for key in self._store.find({'key': {'$exists': True}}):
yield (key['key'], key['value'])
def values(self):
loads = self.loads
for value in self._store.distinct('key'):
yield loads(value['value'])
def test_connection(self):
c = Connection(host, port)
self.assertTrue(c.auto_start_request)
self.assertEqual(None, c.max_pool_size)
self.assertFalse(c.slave_okay)
self.assertFalse(c.safe)
self.assertEqual({}, c.get_lasterror_options())
# Connection's writes are unacknowledged by default
doc = {"_id": ObjectId()}
coll = c.pymongo_test.write_concern_test
coll.drop()
coll.insert(doc)
coll.insert(doc)
c = Connection("mongodb://%s:%s/?safe=true" % (host, port))
self.assertTrue(c.safe)
# To preserve legacy Connection's behavior, max_size should be None.
# Pool should handle this without error.
self.assertEqual(None, c._MongoClient__pool.max_size)
c.end_request()
# Connection's network_timeout argument is translated into
# socketTimeoutMS
self.assertEqual(123, Connection(
host, port, network_timeout=123)._MongoClient__net_timeout)
for network_timeout in 'foo', 0, -1:
self.assertRaises(
ConfigurationError,
Connection, host, port, network_timeout=network_timeout)
def init(self):
logger.info("[Trending broker] I init the %s server connection to %s:%s (%s)" % (self.get_name(), self.backend, self.uri, self.replica_set))
if self.replica_set:
self.con = ReplicaSetConnection(self.uri, replicaSet=self.replica_set, safe=True)
else:
# Old versions of pymongo do not known about fsync
if ReplicaSetConnection:
self.con = Connection(self.uri, safe=True)
else:
self.con = Connection(self.uri, safe=True)
# Open a connection
self.db = getattr(self.con, self.database)
self.col = self.db['trending']
def _open(self):
"""
See mongodb uri documentation:
http://www.mongodb.org/display/DOCS/Connections
"""
client = self.connection.client
hostname = client.hostname or DEFAULT_HOST
authdb = dbname = client.virtual_host
if dbname in ["/", None]:
dbname = "kombu_default"
authdb = "admin"
if not client.userid:
hostname = hostname.replace('/' + client.virtual_host, '/')
else:
hostname = hostname.replace('/' + client.virtual_host,
'/' + authdb)
mongo_uri = 'mongodb://' + hostname
# At this point we expect the hostname to be something like
# (considering replica set form too):
#
# mongodb://[username:password@]host1[:port1][,host2[:port2],
# ...[,hostN[:portN]]][/[?options]]
mongoconn = Connection(host=mongo_uri, ssl=client.ssl)
database = getattr(mongoconn, dbname)
version = mongoconn.server_info()['version']
if tuple(map(int, version.split('.')[:2])) < (1, 3):
raise NotImplementedError(
'Kombu requires MongoDB version 1.3+ (server is {0})'.format(
version))
self.db = database
col = database.messages
col.ensure_index([('queue', 1), ('_id', 1)], background=True)
if 'messages.broadcast' not in database.collection_names():
capsize = (client.transport_options.get('capped_queue_size')
or 100000)
database.create_collection('messages.broadcast',
size=capsize, capped=True)
self.bcast = getattr(database, 'messages.broadcast')
self.bcast.ensure_index([('queue', 1)])
self.routing = getattr(database, 'messages.routing')
self.routing.ensure_index([('queue', 1), ('exchange', 1)])
return database
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = Connection(primary, use_greenlets=use_greenlets)
secondary_cx = Connection(secondary, use_greenlets=use_greenlets)
rsc = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
def _open(self):
conninfo = self.connection.client
mongoconn = Connection(host=conninfo.hostname, port=conninfo.port)
dbname = conninfo.virtual_host
version = mongoconn.server_info()["version"]
if tuple(map(int, version.split("."))) < (1, 3):
raise NotImplementedError(
"Kombu requires MongoDB version 1.3+, but connected to %s" % (
version, ))
if not dbname or dbname == "/":
dbname = "kombu_default"
database = getattr(mongoconn, dbname)
col = database.messages
col.ensure_index([("queue", 1)])
return col
def main():
connection = Connection()
runner = BenchmarkRunner(10000, connection.server_info()["gitVersion"])
runner.run_benchmark(Encode(small, "small"))
runner.run_benchmark(Encode(medium, "medium"))
runner.run_benchmark(Encode(large, "large"))
runner.run_benchmark(Decode(BSON.from_dict(small), "small"))
runner.run_benchmark(Decode(BSON.from_dict(medium), "medium"))
runner.run_benchmark(Decode(BSON.from_dict(large), "large"))
runner.run_benchmark(Insert(connection.benchmark, medium, "medium"))
runner.run_benchmark(FindOne(connection.benchmark.medium_no_index, {"x": 5000}, "medium"))
def _open(self):
conninfo = self.connection.client
mongoconn = Connection(host=conninfo.hostname, port=conninfo.port)
dbname = conninfo.virtual_host
version = mongoconn.server_info()["version"]
if tuple(map(int, version.split("."))) < (1, 3):
raise NotImplementedError(
"Kombu requires MongoDB version 1.3+, but connected to %s" %
(version, ))
if not dbname or dbname == "/":
dbname = "kombu_default"
database = getattr(mongoconn, dbname)
col = database.messages
col.ensure_index([("queue", 1)])
return col
def connect(self):
"""Connects to Mongo database, loads options and set connectors.
@raise CuckooReportError: if unable to connect.
"""
host = self.options.get("host", "127.0.0.1")
port = self.options.get("port", 27017)
try:
self.conn = Connection(host, port)
self.db = self.conn.cuckoo
self.fs = GridFS(self.db)
except TypeError:
raise CuckooReportError("Mongo connection port must be integer")
except ConnectionFailure:
raise CuckooReportError("Cannot connect to MongoDB")
def test_constants(self):
Connection.HOST = self.host
Connection.PORT = self.port
self.assertTrue(Connection())
Connection.HOST = "somedomainthatdoesntexist.org"
Connection.PORT = 123456789
assertRaisesExactly(ConnectionFailure,
Connection,
connectTimeoutMS=600)
self.assertTrue(Connection(self.host, self.port))
Connection.HOST = self.host
Connection.PORT = self.port
self.assertTrue(Connection())
def setUp(self):
self.host = os.environ.get("DB_IP", "localhost")
self.port = int(os.environ.get("DB_PORT", 27017))
default_connection = Connection(self.host, self.port)
no_auto_connection = Connection(self.host, self.port,
auto_start_request=False)
pooled_connection = Connection(self.host, self.port,
pool_size=10, timeout=-1)
no_auto_pooled_connection = Connection(self.host, self.port,
pool_size=10, timeout=-1,
auto_start_request=False)
self.default_db = default_connection["pymongo_test"]
self.pooled_db = pooled_connection["pymongo_test"]
self.no_auto_db = no_auto_connection["pymongo_test"]
self.no_auto_pooled_db = no_auto_pooled_connection["pymongo_test"]
class TestDictConfig(unittest.TestCase):
def setUp(self):
""" Create an empty database that could be used for logging """
self.db_name = '_mongolog_test_dict'
self.collection_name = 'log_test'
self.configDict = {
'version': 1,
'handlers': {
'mongo': {
'class': 'mongolog.handlers.MongoHandler',
'db': self.db_name,
'collection': self.collection_name,
'level': 'INFO'
}
},
'root': {
'handlers': ['mongo'],
'level': 'INFO'
}
}
self.conn = Connection('localhost')
self.conn.drop_database(self.db_name)
def testLoggingDictConfiguration(self):
dictConfig(self.configDict)
log = logging.getLogger('dict_example')
log.debug('testing dictionary config')
r = self.conn[self.db_name][self.collection_name]
message = r.find_one({'level': 'debug', 'msg': 'dict_example'})
self.assertEquals(
message, None,
"Logger put debug message in when info level handler requested")
log.info('dict_example')
message = r.find_one({'level': 'info', 'msg': 'dict_example'})
self.assertNotEquals(message, None,
"Logger didn't insert message into database")
self.assertEquals(
message['msg'], 'dict_example',
"Logger didn't insert correct message into database")
def tearDown(self):
""" Drop used database """
self.conn.drop_database(self.db_name)
def generate_doc_content():
generate_random_content()
conn = Connection()
conn.drop_database('peduli')
print("Adding users.", sep="")
for i in range(50):
create_new_user()
print(".", end="")
print(".\nAdding contents.", end="")
for i in range(500):
create_new_content(i)
print(".", end="")
sys.stdout.flush()
print(".")
def init_app(self, app, **kwargs):
"""Initializes `app`, a :class:`~flask.Flask` application, for use with
the specified configuration variables. Keyword arguments passed to this
override the configuration options.
"""
options = {
'max_pool_size': app.config.get('MONGO_MAX_POOL_SIZE', 10),
'network_timeout': app.config.get('MONGO_NETWORK_TIMEOUT', None),
'tz_aware': app.config.get('MONGO_TZ_AWARE', False),
'slave_okay': app.config.get('MONGO_SLAVE_OKAY', False),
'safe': app.config.get('MONGO_GETLASTERROR', False),
'fsync': app.config.get('MONGO_GETLASTERROR_FSYNC', None),
'j': app.config.get('MONGO_GETLASTERROR_J', None),
'w': app.config.get('MONGO_GETLASTERROR_W', None),
'wtimeout': app.config.get('MONGO_GETLASTERROR_W_TIMEOUT', None),
'replicaset': app.config.get('MONGO_REPLICA_SET', None),
}.update(kwargs)
self.app = app
self.hosts = app.config.get('MONGO_HOSTS', "mongodb://localhost:27017")
self.connection = BaseConnection(self.hosts, options)
@app.teardown_request
def free_sockets(response):
# release thread connection to pool so socket is reclaimed
self.connection.end_request()
return response
for model, indices in _indices.iteritems():
for index in indices:
index.ensure(model.query)
def get_connection(self):
"""Connect to the MongoDB server."""
from pymongo.connection import Connection
if self._connection is None:
self._connection = Connection(self.host, self.port)
return self._connection
def generate_doc_content():
generate_random_content()
conn = Connection()
conn.drop_database("peduli")
print("Adding users.", sep="")
for i in range(50):
create_new_user()
print(".", end="")
print(".\nAdding contents.", end="")
for i in range(500):
create_new_content(i)
print(".", end="")
sys.stdout.flush()
print(".")
def import_cdr_freeswitch_mongodb(shell=False):
#TODO : dont use the args here
# Browse settings.MG_IMPORT and for each IP check if the IP exist
# in our Switch objects. If it does we will connect to that Database
# and import the data as we do below
print_shell(shell, "Starting the synchronization...")
#loop within the Mongo CDR Import List
for ipaddress in settings.MG_IMPORT:
data = chk_ipaddress(ipaddress)
ipaddress = data['ipaddress']
switch = data['switch']
#Connect on MongoDB Database
host = settings.MG_IMPORT[ipaddress]['host']
port = settings.MG_IMPORT[ipaddress]['port']
db_name = settings.MG_IMPORT[ipaddress]['db_name']
try:
connection = Connection(host, port)
DBCON = connection[db_name]
except ConnectionFailure, e:
sys.stderr.write("Could not connect to MongoDB: %s - %s" % \
(e, ipaddress))
sys.exit(1)
#Connect to Mongo
importcdr_handler = DBCON[settings.MG_IMPORT[ipaddress]['collection']]
#Start import for this mongoDB
func_importcdr_aggregate(shell, importcdr_handler, switch, ipaddress)
def init_app(self, app, **kwargs):
"""Initializes `app`, a :class:`~flask.Flask` application, for use with
the specified configuration variables. Keyword arguments passed to this
override the configuration options.
"""
options = {
'max_pool_size': app.config.get('MONGO_MAX_POOL_SIZE', 10),
'network_timeout': app.config.get('MONGO_NETWORK_TIMEOUT', None),
'tz_aware': app.config.get('MONGO_TZ_AWARE', False),
'slave_okay': app.config.get('MONGO_SLAVE_OKAY', False),
'safe': app.config.get('MONGO_GETLASTERROR', False),
'fsync': app.config.get('MONGO_GETLASTERROR_FSYNC', None),
'j': app.config.get('MONGO_GETLASTERROR_J', None),
'w': app.config.get('MONGO_GETLASTERROR_W', None),
'wtimeout': app.config.get('MONGO_GETLASTERROR_W_TIMEOUT', None),
'replicaset': app.config.get('MONGO_REPLICA_SET', None),
}.update(kwargs)
self.app = app
self.hosts = app.config.get('MONGO_HOSTS', "mongodb://localhost:27017")
self.connection = BaseConnection(self.hosts, options)
@app.teardown_request
def free_sockets(response):
# release thread connection to pool so socket is reclaimed
self.connection.end_request()
return response
for model, indices in _indices.iteritems():
for index in indices:
index.ensure(model.query)
def mongo_connection(klass):
""" returns a pointer to the DB"""
if not getattr(klass, 'MONGO_CONNECTION', None):
from pymongo.connection import Connection
connection = Connection(settings.MONGO_HOST, settings.MONGO_PORT)
klass.MONGO_CONNECTION = connection
return klass.MONGO_CONNECTION
def connect(self,
database,
timezone=None,
cache_size=0,
auto_ensure=True,
*args,
**kwds):
''' `connect` is a thin wrapper around __init__ which creates the
database connection that the session will use.
:param database: the database name to use. Should be an instance of \
:class:`basestring`
:param safe: The value for the "safe" parameter of the Session \
init function
:param auto_ensure: Whether to implicitly call ensure_indexes on all write \
operations.
:param args: arguments for :class:`pymongo.connection.Connection`
:param kwds: keyword arguments for :class:`pymongo.connection.Connection`
'''
safe = kwds.get('safe', False)
if 'safe' in kwds:
del kwds['safe']
if timezone is not None:
kwds['tz_aware'] = True
conn = Connection(*args, **kwds)
db = conn[database]
return Session(db,
timezone=timezone,
safe=safe,
cache_size=cache_size,
auto_ensure=auto_ensure)
def readSHPPoint(append):
fileP = 'F:\\project\\Kalium_web\\shapefile\\co99_d00.shp'
sf = shapefile.Reader(fileP)
shapeRecs = sf.shapeRecords()
mongodb_server='127.0.0.1'
mongodb_port = 27017
mongodb_collection ='test1'
mongodb_db = 'gisdb'
connection = Connection(mongodb_server, mongodb_port)
print 'Getting database %s' % mongodb_db
db = connection[mongodb_db]
print 'Getting the collection %s' % mongodb_collection
collection = db[mongodb_collection]
if append == False:
print 'Removing features from the collection...'
collection.remove({})
print 'Starting loading features...'
for shaperec in shapeRecs:
mongofeat = {}
#'{x='',y=''}'
strX = "%.3f" % shaperec.shape.points[0][0]
strY = "%.3f" % shaperec.shape.points[0][1]
mongogeom = '{x="'+strX+'",y="'+strY+'"}'
print mongogeom
mongofeat['geom'] = mongogeom
mongofeat['name'] = shaperec.record[1]
mongofeat['tmp'] = shaperec.record[2]
collection.insert(mongofeat)
#create 2d index
collection.create_index([("geom", pymongo.GEO2D)])
def main():
parser = OptionParser(
usage='mtop.py [options]\nSee also: https://github.com/beaufour/mtop')
parser.add_option('-s',
'--server',
dest='server',
default='localhost',
help='connect to mongo on SERVER',
metavar='SERVER')
parser.add_option('-d',
'--delay',
dest='delay',
type=int,
default=1000,
help='update every MS',
metavar='MS')
(options, _) = parser.parse_args()
try:
if hasattr(
pymongo, 'version_tuple'
) and pymongo.version_tuple[0] >= 2 and pymongo.version_tuple[1] >= 4:
from pymongo import MongoClient
from pymongo.read_preferences import ReadPreference
connection = MongoClient(host=options.server,
read_preference=ReadPreference.SECONDARY)
else:
from pymongo.connection import Connection
connection = Connection(options.server, slave_okay=True)
except AutoReconnect, ex:
print 'Connection to %s failed: %s' % (options.server, str(ex))
return -1
def setUp(self):
"""
set up DAS core module
"""
debug = 0
config = deepcopy(das_readconfig())
logger = PrintManager('TestDASMongocache', verbose=debug)
config['logger'] = logger
config['verbose'] = debug
dburi = config['mongodb']['dburi']
connection = Connection(dburi)
connection.drop_database('das')
dasmapping = DASMapping(config)
config['dasmapping'] = dasmapping
self.dasmongocache = DASMongocache(config)
def main():
parser = SafeConfigParser()
parser.read('settings.ini')
connection = Connection(parser.get('mongodb', 'server'))
db = None
exec('db = connection.' + parser.get('mongodb', 'db'))
ai = mc(db, "markov", exp=1.2)
deck = []
for i in range(100):
c = ai.get()
if (isinstance(c, type(""))):
deck.append(c)
deck.sort()
deck_clean = bag(lambda: 0)
for c in deck:
deck_clean[c] += 1
for c in set(deck):
print("%2i %s" % (deck_clean[c], c))
def test_repr(self):
self.skip()
connection = Connection.paired(self.left, self.right)
self.assertEqual(
repr(connection), "Connection(['%s:%s', '%s:%s'])" %
(self.left[0], self.left[1], self.right[0], self.right[1]))
def _create_mongo_connection(self):
host_uri = \
'mongodb://%s' % (",".join(["%s:%s" % h for h in self.url_nodelist]))
log.info("[MongoDBGridFS] Host URI: %s" % host_uri)
params = {}
params['slaveok'] = self.url_options.get("slaveok", False)
if self.url_options.has_key("replicaset"):
params['replicaset'] = self.url_options["replicaset"] or ""
conn = Connection(host_uri, **params)
db = conn[self.url_database]
if self.url_username:
log.info("[MongoDBGridFS] Attempting to authenticate %s/%s " %
(self.url_username, self.url_password))
if not db.authenticate(self.url_username, self.url_password):
raise InvalidCacheBackendError(
'Cannot authenticate to MongoDB.')
collection = db["%s.files" % self.url_collection]
collection.ensure_index([("namespace", ASCENDING),
("filename", ASCENDING)],
unique=True)
collection.ensure_index([("namespace", ASCENDING)])
return (db, GridFS(db, self.url_collection))
def mongodb2shape(mongodb_server, mongodb_port, mongodb_db, mongodb_collection, output_shape, query_filter):
"""Convert a mongodb collection (all elements must have same attributes) to a shapefile"""
print ' Converting a mongodb collection to a shapefile '
connection = Connection(mongodb_server, mongodb_port)
print 'Getting database MongoDB %s...' % mongodb_db
db = connection[mongodb_db]
print 'Getting the collection %s...' % mongodb_collection
collection = db[mongodb_collection]
print 'Exporting %s elements in collection to shapefile...' % collection.count()
drv = ogr.GetDriverByName("ESRI Shapefile")
ds = drv.CreateDataSource(output_shape)
lyr = ds.CreateLayer('test', None, ogr.wkbPolygon)
print 'Shapefile %s created...' % ds.name
cursor = collection.find(query_filter)
# define the progressbar
pbar = ProgressBar(collection.count()).start()
k=0
# iterate the features in the collection and copy them to the shapefile
# for simplicity we export only the geometry to the shapefile
# if we would like to store also the other fields we should have created a metadata element with fields datatype info
for element in cursor:
element_geom = element['geom']
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetGeometry(ogr.CreateGeometryFromWkt(element_geom))
lyr.CreateFeature(feat)
feat.Destroy()
k = k + 1
pbar.update(k)
pbar.finish()
print '%s features loaded in shapefile from MongoDb.' % lyr.GetFeatureCount()
def __init__(self):
try:
self.connection = Connection()
#define database name
self.db = self.connection.todo_db
except Exception, e:
raise Exception("Error!! Please check db conenction: %s" % e)
def _connect(self):
self._conn = None
try:
if len(self.master_args) == 2:
self._conn = Connection.paired(
(str(self.master_args[0]['host']), int(self.master_args[0]['port'])),
(str(self.master_args[1]['host']), int(self.master_args[1]['port'])),
pool_size=int(self.master_args[0]['query'].get('pool_size','16')))
else:
if self.master_args:
try:
network_timeout = self.master_args[0]['query'].get('network_timeout')
if network_timeout is not None:
network_timeout = float(network_timeout)
master = Connection(str(self.master_args[0]['host']), int(self.master_args[0]['port']),
pool_size=int(self.master_args[0]['query'].get('pool_size','16')),
network_timeout=network_timeout)
###authenticating when the db requires it to be done.
if self.master_args[0].get('username') and self.master_args[0].get('password'):
db = database.Database(master, self.master_args[0]['path'][1:])
db.authenticate(self.master_args[0]['username'],
self.master_args[0]['password'])
###
except:
if self.slave_args:
log.exception('Cannot connect to master: %s will use slave: %s' % (self.master_args, self.slave_args))
# and continue... to use the slave only
master = None
else:
raise
else:
log.info('No master connection specified, using slaves only: %s' % self.slave_args)
master = None
if self.slave_args:
slave = []
for a in self.slave_args:
network_timeout = a['query'].get('network_timeout')
if network_timeout is not None:
network_timeout = float(network_timeout)
slave.append(
Connection(str(a['host']), int(a['port']),
pool_size=int(a['query'].get('pool_size','16')),
slave_okay=True,
network_timeout=network_timeout,
)
)
if master:
self._conn = MasterSlaveConnection(master, slave)
else:
self._conn = slave[0]
else:
self._conn = master
except:
log.exception('Cannot connect to %s %s' % (self.master_args, self.slave_args))
else:
#log.info('Connected to %s %s' % (self.master_args, self.slave_args))
pass
return self._conn
def _connect_mongodb(settings, mongodb):
# type: (Settings, MongoDB) -> None
if Connection is None:
raise Exception(
'Could not initialize MongoDB (Python-Modules are missing)')
mongodb.connection = Connection(*_mongodb_local_connection_opts(settings))
mongodb.db = mongodb.connection.__getitem__(os.environ['OMD_SITE'])
def test_mongo_replica_set_client(self):
c = Connection(pair)
ismaster = c.admin.command('ismaster')
if 'setName' in ismaster:
setname = str(ismaster.get('setName'))
else:
raise SkipTest("Not connected to a replica set.")
m = MongoReplicaSetClient(pair, replicaSet=setname, w=0)
coll = m.pymongo_test.write_concern_test
coll.drop()
doc = {"_id": ObjectId()}
coll.insert(doc)
self.assertTrue(coll.insert(doc, safe=False))
self.assertTrue(coll.insert(doc, w=0))
self.assertTrue(coll.insert(doc))
self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
self.assertRaises(OperationFailure, coll.insert, doc, w=1)
m = MongoReplicaSetClient(pair, replicaSet=setname)
coll = m.pymongo_test.write_concern_test
self.assertTrue(coll.insert(doc, safe=False))
self.assertTrue(coll.insert(doc, w=0))
self.assertRaises(OperationFailure, coll.insert, doc)
self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
self.assertRaises(OperationFailure, coll.insert, doc, w=1)
m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s" % (pair, setname))
self.assertTrue(m.safe)
coll = m.pymongo_test.write_concern_test
self.assertRaises(OperationFailure, coll.insert, doc)
m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s;w=0" % (pair, setname))
self.assertFalse(m.safe)
coll = m.pymongo_test.write_concern_test
self.assertTrue(coll.insert(doc))
def test_postProcessing():
mongodb_connection = Connection('localhost', 27017)
crowds_db = mongodb_connection.set_ats_new
crowds_collection = crowds_db.Crowd
i = 1
for crowd in crowds_collection.find():
for k in ['start', 'end']:
if crowd[k]:
if type(crowd[k]) == type(1.0):
crowd[k] = dt.utcfromtimestamp(crowd[k])
else:
print 'nop', crowd['_id']
else:
crowd[k] = dt(2011, 6, 1)
end = crowd['end']
for user in crowd['users']:
user['id'] = int(user['id'])
for h in user['history']:
for x in xrange(2):
if h[x]:
if type(h[x]) == type(1.0):
h[x] = dt.utcfromtimestamp(h[x])
else:
h[x] = end
print i, crowd['_id']
i += 1
crowds_collection.save(crowd)
def test_iteration(self):
connection = Connection(self.host, self.port)
def iterate():
[a for a in connection]
self.assertRaises(TypeError, iterate)
def mongo_upgrade():
"""Migrate mongodb schema and data."""
# Read reporting.conf to fetch mongo configuration.
config = Config(os.path.join("..", "..", "conf", "reporting.conf"))
# Run migration only if mongo is enabled as reporting module.
if config.mongodb.enabled:
host = config.mongodb.get("host", "127.0.0.1")
port = config.mongodb.get("port", 27017)
print "Mongo reporting is enabled, strarting mongo data migration."
# Connect.
try:
conn = Connection(host, port)
db = conn.cuckoo
except TypeError:
print "Mongo connection port must be integer"
sys.exit()
except ConnectionFailure:
print "Cannot connect to MongoDB"
sys.exit()
# Check for schema version and create it.
if "cuckoo_schema" in db.collection_names():
print "Mongo schema version not expected"
sys.exit()
else:
db.cuckoo_schema.save({"version": mongo_revision})
else:
print "Mongo reporting module not enabled, skipping mongo migration."
