def store_feeds(feeds, collection):
'store the feeds in mongodb '
from pymongo.errors import CollectionInvalid
from pymongo.collection import Collection
from pymongo.connection import Connection
con = Connection(DB)
from pymongo.database import Database
db = Database(con, 'articles')
col = None
try:
col = db.create_collection(collection)
except CollectionInvalid as e:
col = Collection(db, collection)
for feed in feeds:
if 'title' in feed:
cursor = col.find({'title':'%s' % feed['title']})
if cursor.count() > 0:
continue
elif 'source' in feed:
cursor = col.find({'source':'%s' % feed['source']})
if cursor.count() > 0:
continue
col.save(feed)
def test_create_collection(self):
db = Database(self.client, "pymongo_test")
db.test.insert({"hello": "world"})
self.assertRaises(CollectionInvalid, db.create_collection, "test")
db.drop_collection("test")
self.assertRaises(TypeError, db.create_collection, 5)
self.assertRaises(TypeError, db.create_collection, None)
self.assertRaises(InvalidName, db.create_collection, "coll..ection")
test = db.create_collection("test")
test.save({"hello": u"world"})
self.assertEqual(db.test.find_one()["hello"], "world")
self.assertTrue(u"test" in db.collection_names())
db.drop_collection("test.foo")
db.create_collection("test.foo")
self.assertTrue(u"test.foo" in db.collection_names())
expected = {}
if version.at_least(self.client, (2, 7, 0)):
# usePowerOf2Sizes server default
expected["flags"] = 1
result = db.test.foo.options()
# mongos 2.2.x adds an $auth field when auth is enabled.
result.pop("$auth", None)
self.assertEqual(result, expected)
self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
def binary_contents_test(self):
db = Database(self._get_connection(), "pymongo_test")
test = db.create_collection("test_binary")
import os
import bson
obj = os.urandom(1024)
test.save({"hello": bson.Binary(obj)})
db.drop_collection("test_binary")
def test_collection_names(self):
db = Database(self.connection, "pymongo_test")
db.test.save({"dummy": u"object"})
db.test.mike.save({"dummy": u"object"})
colls = db.collection_names()
self.assert_("test" in colls)
self.assert_("test.mike" in colls)
for coll in colls:
self.assert_("$" not in coll)
def __init__(self, config):
database = config['mongo_db_name']
if 'mongo_login' in config:
passwordfile = config['mongo_login']
else:
passwordfile = None
connection = MongoDB.connect(config)
Database.__init__(self, connection, database)
if passwordfile is not None and exists(passwordfile):
password = yaml.load(open(passwordfile, 'rU'))
self.authenticate(password['mongo_user'], password['mongo_password'])
def test_collection_names(self):
db = Database(self.client, "pymongo_test")
db.test.save({"dummy": u"object"})
db.test.mike.save({"dummy": u"object"})
colls = db.collection_names()
self.assertTrue("test" in colls)
self.assertTrue("test.mike" in colls)
for coll in colls:
self.assertTrue("$" not in coll)
colls_without_systems = db.collection_names(False)
for coll in colls_without_systems:
self.assertTrue(not coll.startswith("system."))
def get_db(is_local_deployed=False):
if is_local_deployed:
#for local launch
connection = Connection()
db = connection.wiki
return db
else:
# for dotcloud launch
mongodb_info = json.load(file("/home/dotcloud/environment.json"))
print mongodb_info
connection = Connection(mongodb_info["DOTCLOUD_DATA_MONGODB_HOST"], int(mongodb_info["DOTCLOUD_DATA_MONGODB_PORT"]))
database = Database(connection, "wiki")
database.authenticate("xinz", "hellopy")
db = connection.wiki
return db
def test4(self):
db = Database(self._get_connection(), "pymongo_test")
test = db.create_collection("test_4")
try:
for i in range(5):
name = "test %d" % (i)
test.save({ "user_id": i, "name": name, "group_id" : i % 10, "posts": i % 20})
test.create_index("user_id")
for i in xrange(6):
for r in test.find( { "group_id": random.randint(0,10) } ):
print "Found: %s " % (r)
finally:
db.drop_collection("test_4")
class MongoopTrigger(BaseTrigger):
def __init__(self, *args, **kwargs):
super(MongoopTrigger, self).__init__(*args, **kwargs)
database = self.params.get('database', 'mongoop')
collection = self.params.get('collection', 'history')
self.db = Database(self.mongoop.conn, database)
self.collection = self.db.get_collection(collection)
self.collection.create_index([('opid', DESCENDING)],
unique=True,
background=True)
def op_nok(self, operations):
try:
if operations:
self.collection.insert_many(operations)
except Exception as e:
logging.error('unable to bulk operations :: {} :: {}'.format(
self.name, e))
return False
else:
logging.info('run :: {} :: bulk insert {} operations'.format(
self.name, len(operations)))
return True
def __init__(self):
db_server_ip = conf("config.db_server_ip")
db_server_port = conf("config.db_server_port")
self.__connection = Connection(host=db_server_ip, port=db_server_port)
db_name = conf("config.db_name")
self.__db = Database(connection= self.__connection, name=db_name)
def borrarindicesevitardups():
# Conexion a local
client = MongoClient()
try:
# The ismaster command is cheap and does not require auth.
client.admin.command('ismaster')
db = Database(client=client,
name=ConfiguracionGlobal.MONGODB)
except ConnectionFailure:
print("Server not available")
# Limpiado tablas e indices con pymongo
for collname in db.collection_names():
coll = Collection(db, name=collname)
coll.drop_indexes()
client.close()
def connect(conf, prod):
log.info('setting up mongo connection')
url = conf.get('mongo', 'url')
port = conf.getint('mongo', 'port')
db = conf.get('mongo', 'db')
conn = Connection(url, port)
db = Database(conn, db)
if (prod):
log.info('authenticating mongo connection')
username = conf.get('mongo', 'username')
pwd = conf.get('mongo', 'password')
db.authenticate(username, pwd)
return db
def __init__(self, *args, **kwargs):
super(MongoopTrigger, self).__init__(*args, **kwargs)
database = self.params.get('database', 'mongoop')
collection = self.params.get('collection', 'history')
self.db = Database(self.mongoop.conn, database)
self.collection = self.db.get_collection(collection)
self.collection.create_index([('opid', DESCENDING)], unique=True, background=True)
def __init__(self, conn_string, database, trace=False):
if not conn_string.startswith('mongodb://'):
conn_string = 'mongodb://' + conn_string
self.connection = Connection(conn_string)
self.database = Database(self.connection, database)
self.trace = trace
self.collections = []
self.obj_infos = set()
self._cache = {}
def store_feeds(feeds, collection):
'store the feeds in mongodb '
from pymongo.errors import CollectionInvalid
from pymongo.collection import Collection
from pymongo.connection import Connection
con = Connection(DB)
from pymongo.database import Database
db = Database(con, 'queries')
col = None
try:
col = db.create_collection(collection)
except CollectionInvalid as e:
col = Collection(db, collection)
for feed in feeds:
existed = col.find_one({'query':feed}, {'$exists':'true'})
if not existed:
col.save({'query':feed})
def conn(self, line):
"""Conenct to mongo database in ipython shell.
Examples::
%mongo_connect
%mongo_connect mongodb://hostname:port
"""
if line:
uri = line
else:
uri = 'mongodb://127.0.0.1:27017'
self._conn = MongoClient(uri)
# add db and collection property to object for autocomplete.
for db in self._conn.database_names():
setattr(self._conn, db, Database(self._conn, db))
_db = Database(self._conn, db)
for collection in _db.collection_names():
# [TODO] change eval to other!!
setattr(eval('self._conn.'+db), collection,
Collection(_db, collection))
return self._conn
def test_get_collection(self):
db = Database(self.client, "pymongo_test")
codec_options = CodecOptions(
tz_aware=True, uuid_representation=JAVA_LEGACY)
write_concern = WriteConcern(w=2, j=True)
coll = db.get_collection(
'foo', codec_options, ReadPreference.SECONDARY, write_concern)
self.assertEqual('foo', coll.name)
self.assertEqual(codec_options, coll.codec_options)
self.assertEqual(JAVA_LEGACY, coll.uuid_subtype)
self.assertEqual(ReadPreference.SECONDARY, coll.read_preference)
self.assertEqual(write_concern.document, coll.write_concern)
pref = Secondary([{"dc": "sf"}])
coll = db.get_collection('foo', read_preference=pref)
self.assertEqual(pref.mode, coll.read_preference)
self.assertEqual(pref.tag_sets, coll.tag_sets)
self.assertEqual(db.codec_options, coll.codec_options)
self.assertEqual(db.uuid_subtype, coll.uuid_subtype)
self.assertEqual(db.write_concern, coll.write_concern)
def test_create_collection(self):
db = Database(self.client, "pymongo_test")
db.test.insert_one({"hello": "world"})
self.assertRaises(CollectionInvalid, db.create_collection, "test")
db.drop_collection("test")
self.assertRaises(TypeError, db.create_collection, 5)
self.assertRaises(TypeError, db.create_collection, None)
self.assertRaises(InvalidName, db.create_collection, "coll..ection")
test = db.create_collection("test")
self.assertTrue(u("test") in db.collection_names())
test.insert_one({"hello": u("world")})
self.assertEqual(db.test.find_one()["hello"], "world")
db.drop_collection("test.foo")
db.create_collection("test.foo")
self.assertTrue(u("test.foo") in db.collection_names())
self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
def init_connection(self):
if hasattr(self, 'app'):
config = self.app.config
else:
config = self._default_config
connection = Connection(
host=config.get('MONGODB_HOST'),
port=config.get('MONGODB_PORT'),
slave_okay=config.get('MONGODB_SLAVE_OKAY')
)
database = Database(connection, config.get('MONGODB_DATABASE'))
if config.get('MONGODB_USERNAME') is not None:
database.authenticate(
config.get('MONGODB_USERNAME'),
config.get('MONGODB_PASSWORD')
)
return connection, database
def load_feature_vectors_and_classes(db_name):
"""
:param db_name: name of database to use
:return:
- list of feature vectors
- dictionary where the keys are the class labels and the values are dictionaries of the form
{start: <integer>, end: <integer>}
"""
print 'Loading feature vectors and target classes...'
db = Database(MongoClient(), db_name)
collection_names = db.collection_names()
if not ('naive_bayes' in collection_names and 'feature_vectors' in collection_names):
print 'Database missing collections needed to train classifier on.'
exit()
target_classes = Collection(db, 'naive_bayes').find_one({'type': 'classes'})
if 'classes' not in target_classes:
raise KeyError("'target_classes' must contain a 'classes' key.")
feature_vectors = list(Collection(db, 'feature_vectors').find())
return feature_vectors, target_classes['classes']
def test_create_collection(self):
db = Database(self.connection, "pymongo_test")
db.test.insert({"hello": "world"})
self.assertRaises(CollectionInvalid, db.create_collection, "test")
db.drop_collection("test")
self.assertRaises(TypeError, db.create_collection, 5)
self.assertRaises(TypeError, db.create_collection, None)
self.assertRaises(InvalidName, db.create_collection, "coll..ection")
test = db.create_collection("test")
test.save({"hello": u"world"})
self.assertEqual(db.test.find_one()["hello"], "world")
self.assert_(u"test" in db.collection_names())
db.drop_collection("test.foo")
db.create_collection("test.foo")
self.assert_(u"test.foo" in db.collection_names())
self.assertEqual(db.test.foo.options(), {})
self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
def test2(self):
db = Database(self._get_connection(), "pymongo_test")
test = db.create_collection("test_2")
try:
for i in range(100):
name = "test %d" % (i)
ret = test.save({"name": name, "group_id" : i % 3, "posts": i % 20})
print "Save Ret: %s" % (ret)
ret = test.update({"posts": 10}, {"$set": {"posts": 100}}, multi=True, safe=True)
#ret = test.update({"posts": 10}, {"$set": {"posts": 100}}, multi=True)
print "Update Ret: %s" % (ret)
test.update({"name": "test 2"}, {"$set": {"posts": 200}})
test.create_index("posts")
test.ensure_index("posts")
for r in test.find({"posts":100}):
print "Found: %s" % (r,)
ret = test.remove({"posts": 1}, safe=True)
print "Remove Ret: %s" % (ret)
groups = test.group(
key={"group_id":1},
condition=None,
initial={"post_sum":0},
reduce="function(obj,prev) {prev.post_sum++;}"
)
for g in groups:
print "Group: %s" % (g,)
for d in test.distinct('posts'):
print "Distinct: %s" % (d,)
if 'reindex' in dir(test):
test.reindex()
test.drop_indexes()
finally:
db.drop_collection("test_2")
def test_collection_names(self):
db = Database(self.client, "pymongo_test")
db.test.insert_one({"dummy": u("object")})
db.test.mike.insert_one({"dummy": u("object")})
colls = db.collection_names()
self.assertTrue("test" in colls)
self.assertTrue("test.mike" in colls)
for coll in colls:
self.assertTrue("$" not in coll)
colls_without_systems = db.collection_names(False)
for coll in colls_without_systems:
self.assertTrue(not coll.startswith("system."))
# Force more than one batch.
db = self.client.many_collections
for i in range(101):
db["coll" + str(i)].insert_one({})
# No Error
try:
db.collection_names()
finally:
self.client.drop_database("many_collections")
def test_get_db(self):
client = MongoClient(host, port)
def make_db(base, name):
return base[name]
self.assertRaises(InvalidName, make_db, client, "")
self.assertRaises(InvalidName, make_db, client, "te$t")
self.assertRaises(InvalidName, make_db, client, "te.t")
self.assertRaises(InvalidName, make_db, client, "te\\t")
self.assertRaises(InvalidName, make_db, client, "te/t")
self.assertRaises(InvalidName, make_db, client, "te st")
self.assertTrue(isinstance(client.test, Database))
self.assertEqual(client.test, client["test"])
self.assertEqual(client.test, Database(client, "test"))
def test_get_db(self):
connection = Connection(self.host, self.port)
def make_db(base, name):
return base[name]
self.assertRaises(InvalidName, make_db, connection, "")
self.assertRaises(InvalidName, make_db, connection, "te$t")
self.assertRaises(InvalidName, make_db, connection, "te.t")
self.assertRaises(InvalidName, make_db, connection, "te\\t")
self.assertRaises(InvalidName, make_db, connection, "te/t")
self.assertRaises(InvalidName, make_db, connection, "te st")
self.assertTrue(isinstance(connection.test, Database))
self.assertEqual(connection.test, connection["test"])
self.assertEqual(connection.test, Database(connection, "test"))
def test_get_db(self):
connection = self._get_connection()
def make_db(base, name):
return base[name]
self.assertRaises(InvalidName, make_db, connection, "")
self.assertRaises(InvalidName, make_db, connection, "te$t")
self.assertRaises(InvalidName, make_db, connection, "te.t")
self.assertRaises(InvalidName, make_db, connection, "te\\t")
self.assertRaises(InvalidName, make_db, connection, "te/t")
self.assertRaises(InvalidName, make_db, connection, "te st")
self.assert_(isinstance(connection.test, Database))
self.assertEqual(connection.test, connection["test"])
self.assertEqual(connection.test, Database(connection, "test"))
connection.close()
def dbncol(client, collection, database='test'):
''' Make a connection to the database and collection given in the arguments.
:param client: a MongoClient instance
:type client: pymongo.MongoClient
:param database: the name of the database to be used. It must be a database name present at the client
:type database: str
:param collection: the database collection to be used. It must be a collection name present in the database
:type collection: str
:return col: the collection to be used
:type: pymongo.collection.Collection
'''
db = Database(client, database)
col = Collection(db, collection)
return col
def resolve_object_ids(self, mongo_object):
if mongo_object is None:
return None
if type(mongo_object) is "dict":
result = { }
for key in mongo_object:
result[key] = self.resolve_object_ids(mongo_object[key]) if key is not "_id" else mongo_object[key]
return result
elif type(mongo_object) is "list":
result = []
for item in mongo_object:
result.append(self.resolve_object_ids(item))
return result
elif type(mongo_object) is "bson.objectid.ObjectId":
return self.get_single(mongo_object)
elif type(mongo_object) is "bson.dbref.DBRef":
return Database.dereference(mongo_object)
def getTweetsTimeSeries(self, mongoDbCollection : Database, stockTicker, fromDate, toDate):
raw_tweets = mongoDbCollection.aggregate([
{"$match": {"stockTicker": stockTicker, "date": {"$gte": fromDate, "$lt": toDate}}},
{"$group": { "_id": {"day": {"$dayOfMonth": "$date"}, "month": {"$month": "$date"}, "year": {"$year": "$date"}},
"count": {"$sum": 1}}}])
raw_tweets = list(raw_tweets)
date_count_tweets = []
# extract tweets to be in list of dictionaries of form date and tweets
# TODO upgrade mongo DB to 3.6 to do a projection with $dateFromParts
for tweet in raw_tweets:
entry = {}
entry['date'] = datetime(year=tweet["_id"]["year"], month=tweet["_id"]["month"], day=tweet["_id"]["day"])
entry['tweets'] = tweet['count']
date_count_tweets.append(entry)
date_count_tweets = sorted(date_count_tweets, key=lambda k: k['date'])
return date_count_tweets
def _conn():
if not nosqlDB.con:
try:
# config = ConfigParser.ConfigParser()
# config.readfp(open(PackageLoader('conf','db.ini')))
nosqlDB.initParas = globalConf.nosqldb
nosqlDB.con = Connection(host=nosqlDB.initParas['host'],
port=nosqlDB.initParas['port'],
max_pool_size=100)
# nosqlDB.con = MongoClient(host=nosqlDB.initParas['host'], port=nosqlDB.initParas['port'], max_pool_size=200)
nosqlDB.db = Database(nosqlDB.con, nosqlDB.initParas['db'])
if nosqlDB.initParas['isauth'] == '1':
nosqlDB.db.auth(nosqlDB.initParas['user'],
nosqlDB.initParas['passwd'])
nosqlDB.gfs = GridFS(nosqlDB.db, 'files')
except BaseException, e:
print e
def find_data(client, database, collection, filters={}):
''' Find the items in the specified database and collection using the filters.
:param client: a MongoClient instance
:type client: pymongo.MongoClient
:param database: the name of the database to be used. It must be a database name present at the client
:type database: str
:param collection: the database collection to be used. It must be a collection name present in the database
:type collection: str
:param filters: the parameters used for filtering the returned data. An empty filter parameter returns the full collection
:type filters: dict
:return: the result of the query
:type: pymongo.cursor.CursorType
'''
db = Database(client, database)
col = Collection(db, collection)
return col.find(filters).batch_size(100)
def get_collection(self,
collection_name,
database=settings.MONGO_DEFAULT_DATABASE):
'''
:param collection_name: collection name
:param database: database name
:return: Collection object of the specified collection name.
'''
try:
mongo_client = self.get_mongo_client()
database_instance = Database(mongo_client, database)
collection = Collection(database=database_instance,
name=collection_name)
return collection
except (PyMongoError, Exception) as e:
logger.error(
f"Error connecting to collection {collection_name} error = {e}"
)
raise e
def parse(cell, config):
parts = [part.strip() for part in cell.split(None, 1)]
if not parts:
return """[ERROR] please enter the db.collection and data
%insert db.collection {data} or [{data}, {data}...]
or
%%insert db.collection
{data} or [{data}, {data} ...]"""
db, col = parts[0].split('.')
if not db and not col:
return """[ERROR] please enter the db and collection name both of all
%insert db.collection {data or data list}
or
%%insert db.collection
{data or data list}"""
_db = Database(config._conn, db)
_col = Collection(_db, col)
data = parts[1].strip() if len(parts) > 1 else ''
return {'collection': _col, 'data': data}
def __init__(self, logger, **kwargs):
self.logger = logger
self.host_array = kwargs.get('hosts')
self.replicaset = kwargs.get('replicaset')
self.db_name = kwargs.get('database_name')
if not self.replicaset:
host_tokens = self.host_array[0].split(':')
hostname = host_tokens[0]
port = 27017
if len(host_tokens) == 2:
port = int(host_tokens[1])
mdb_client = MongoClient(host=hostname, port=port)
self.db = Database(mdb_client, self.db_name)
else:
raise Exception(
'Connect-to-replicaset not yet implemented (but coming soon)')
def calculate_user_storage(db: Database, user_id: ObjectId):
"""
Get storage that the user has used
:param db: Database instance
:param user_id: The user's objectId
:return: Storage size, a list of files
"""
medias_size = 0
file_set = set()
# Get all collections and map data by user id
for col in db.list_collection_names():
data_list = db[col].aggregate([{
'$match': {
'userId': user_id
}
}, {
"$addFields": {
"paths": {
'$reduce': {
'input': "$items.example.medias",
'initialValue': [],
'in': {
'$concatArrays': ["$$value", "$$this.src"]
}
}
}
}
}, {
'$project': {
'paths': 1
}
}, {
'$unwind': '$paths'
}])
# Map all files' addresses
for path_obj in data_list:
if 'paths' in path_obj and path_obj['paths']:
fp = path_obj['paths'][1:]
if os.path.exists(fp) and fp not in file_set:
# Build status for storage
medias_size += os.path.getsize(fp)
file_set.add(fp)
return medias_size, file_set
def __init__(
self,
model_class: Type[T],
database: Database,
col_name: str,
indexes: Optional[list[Union[IndexModel, str]]] = None,
wrap_object_str_id=True,
):
codecs = CodecOptions(
type_registry=TypeRegistry([c() for c in [DecimalCodec]]))
self.collection = database.get_collection(col_name, codecs)
if indexes:
indexes = [
parse_str_index_model(i) if isinstance(i, str) else i
for i in indexes
]
self.collection.create_indexes(indexes)
self.model_class = model_class
self.wrap_object_id = model_class.__fields__[
"id"].type_ == ObjectIdStr and wrap_object_str_id
def test_update_collection_schema(mongo_handler: MongoHandler,
evo_db: Database,
temp_coll_name: str,
scheme_and_data: Tuple[Dict[str, Any]]):
valid_scheme, valid_data, invalid_data = scheme_and_data
collection = evo_db.get_collection(temp_coll_name)
result = mongo_handler.update_collection_schema(
temp_coll_name, valid_scheme)
assert result
with pytest.raises(WriteError):
collection.insert_one(invalid_data)
doc_id = collection.insert_one(valid_data)
assert doc_id is not None
col_name = "imaginary"
result = mongo_handler.update_collection_schema(col_name, valid_scheme)
assert not result
def update_collections(db: Database, refresh: bool) -> None:
"""
If covid or states collections don't exist in db, downloads data, creates collections, and inserts data
:param refresh: If True, download current data and update database
:param db: PyMongo Database in which to update collections
"""
collections = db.list_collection_names()
if COLL_COVID not in collections or refresh:
if refresh:
db[COLL_COVID].drop()
covid_data: JSON = get_covid_data()
add_collection(db, COLL_COVID, covid_data)
fix_covid_data(db, COLL_COVID)
if COLL_STATES not in collections or refresh:
if refresh:
db[COLL_STATES].drop()
states_data: JSON = get_states_data()
add_cumulative_properties(states_data)
add_collection(db, COLL_STATES, states_data)
fix_states_data(db, COLL_STATES)
def __init__(self,
domain,
host="mongodb://localhost:27017",
db="rasa",
username=None,
password=None,
collection="conversations",
event_broker=None):
from pymongo.database import Database
from pymongo import MongoClient
self.client = MongoClient(host,
username=username,
password=password,
# delay connect until process forking is done
connect=False)
self.db = Database(self.client, db)
self.collection = collection
super(MongoTrackerStore, self).__init__(domain, event_broker)
self._ensure_indices()
def encode_model(obj, recursive=False):
if obj is None:
return obj
if isinstance(obj, (Document, EmbeddedDocument)):
out = dict(obj._data)
for k, v in out.items():
if isinstance(v, ObjectId):
if k is None:
del(out[k])
else:
# Unlikely that we'll hit this since ObjectId is always NULL key
out[k] = str(v)
else:
out[k] = encode_model(v)
elif isinstance(obj, ModuleType):
out = None
elif isinstance(obj, groupby):
out = [(gb, list(l)) for gb, l in obj]
elif isinstance(obj, (list)):
out = [encode_model(item) for item in obj]
elif isinstance(obj, (dict)):
out = dict([(k, encode_model(v)) for (k, v) in obj.items()])
elif hasattr(obj, 'isoformat'):
out = obj.isoformat()
elif isinstance(obj, ObjectId):
return None
elif isinstance(obj, (str, unicode, int)):
out = obj
elif isinstance(obj, float):
out = str(obj)
elif isinstance(obj, DBRef):
db = Database(g.db, 'listenitlater')
obj_deref = Database.dereference(db, obj)
out = encode_model(obj_deref)
else:
raise TypeError, "Could not JSON-encode type '%s': %s" % (type(obj), str(obj))
return out
async def get_member(
database: Database, user_id: str, *, force_refresh: bool = False
) -> typing.Optional[models.DiscordMember]:
"""
Get a member from the cache, or from the discord API.
If `force_refresh` is True, the cache is skipped and the entry is updated.
None may be returned if the member object does not exist.
"""
collection = database.get_collection("discord_members")
if force_refresh:
await collection.delete_one({"user": user_id})
# `create_index` creates the index if it does not exist, or passes
# This handles TTL on member objects
await collection.create_index(
"inserted_at",
expireAfterSeconds=60 * 60, # 1 hour
name="inserted_at",
)
result = await collection.find_one({"user": user_id})
if result is not None:
return models.DiscordMember(**json.loads(result["data"]))
member = await _fetch_member_api(user_id)
if not member:
return None
await collection.insert_one({
"user": user_id,
"data": member.json(),
"inserted_at": datetime.datetime.now(tz=datetime.timezone.utc),
})
return member
def test_mongo_session():
"""
Test if mongo is storing data
:return:
"""
from pymongo import MongoClient
from pymongo.database import Database
# Parameters for MongoDB connection
URL = "mongodb"
PORT = 27017
DB_NAME = "rasa"
client = MongoClient(host=URL, port=PORT)
db = Database(client, DB_NAME)
if db["sessions"].count_documents({}) > 0:
print_passed("Session is stored in Mongo DB")
else:
print_failed(
"Nothing found in Mongo, either mongo is not up or chat room input failed"
)
exit(-1)
def __init__(
self,
domain,
host=os.getenv("MONGO_URL") or "mongodb://localhost:27017",
db="rasa",
username=None,
password=None,
auth_source="admin",
collection="conversations",
neo4j_host=os.getenv("NEO4J_URL") or "localhost",
neo4j_http_port=int(os.getenv("NEO4J_PORT") or 7687),
neo4j_user=os.getenv("NEO4J_USER") or None,
neo4j_password=os.getenv("NEO4J_PASSWORD") or None,
event_broker=None,
):
from pymongo.database import Database
from pymongo import MongoClient
self.client = MongoClient(
host,
username=username,
password=password,
authSource=auth_source,
connect=False,
)
try:
self.Tracker4J = Tracker4J.Tracker4J(host=neo4j_host,
port=neo4j_http_port,
user=neo4j_user,
password=neo4j_password)
except:
self.Tracker4J = None
self.db = Database(self.client, db)
self.collection = collection
super().__init__(domain, event_broker)
self._ensure_indices()
def test(cls, db: database.Database):
for name in db.list_collection_names():
print(name)
# def getComputedStressList(examInfo, user, shift=0, pospond=0, db=getDB()):
# # user = examSession.getUser()
# gmt_begin = examInfo.startDate - dt.timedelta(minutes=shift)
# gmt_end = examInfo.endDate + dt.timedelta(minutes=pospond)
# # convert GMT datetime to GMT timestamp
# timestamp_begin = unix_time_secs(gmt_begin)
# timestamp_end = unix_time_secs(gmt_end)
#
# cs_results = list(db[user.getMatrNr() +'_hrvdata'].find({
# '_id.timestamp':{
# '$gt':timestamp_begin,
# '$lt':timestamp_end
# }
# }))
# return cs_results
#
# def getHeartRateList(examInfo, user, shift=0, pospond=0, db=getDB()):
# # user = examSession.getUser()
# gmt_begin = examInfo.startDate - dt.timedelta(minutes=shift)
# gmt_end = examInfo.endDate + dt.timedelta(minutes=pospond)
# # convert GMT datetime to GMT timestamp
# timestamp_begin = unix_time_secs(gmt_begin)
# timestamp_end = unix_time_secs(gmt_end)
#
# hr_results = list(db[user.getMatrNr() + '_heartrate'].find({
# '_id.timestamp':{
# '$gt':timestamp_begin,
# '$lt':timestamp_end
# }
# }))
# return hr_results
def test_collection_names(self):
db = Database(self.client, "pymongo_test")
db.test.insert_one({"dummy": u"object"})
db.test.mike.insert_one({"dummy": u"object"})
colls = db.collection_names()
self.assertTrue("test" in colls)
self.assertTrue("test.mike" in colls)
for coll in colls:
self.assertTrue("$" not in coll)
colls_without_systems = db.collection_names(False)
for coll in colls_without_systems:
self.assertTrue(not coll.startswith("system."))
# Force more than one batch.
db = self.client.many_collections
for i in range(101):
db["coll" + str(i)].insert_one({})
# No Error
try:
db.collection_names()
finally:
self.client.drop_database("many_collections")
def test_drop_collection(self):
db = Database(self.client, "pymongo_test")
self.assertRaises(TypeError, db.drop_collection, 5)
self.assertRaises(TypeError, db.drop_collection, None)
db.test.insert_one({"dummy": u("object")})
self.assertTrue("test" in db.collection_names())
db.drop_collection("test")
self.assertFalse("test" in db.collection_names())
db.test.insert_one({"dummy": u("object")})
self.assertTrue("test" in db.collection_names())
db.drop_collection(u("test"))
self.assertFalse("test" in db.collection_names())
db.test.insert_one({"dummy": u("object")})
self.assertTrue("test" in db.collection_names())
db.drop_collection(db.test)
self.assertFalse("test" in db.collection_names())
db.test.insert_one({"dummy": u("object")})
self.assertTrue("test" in db.collection_names())
db.test.drop()
self.assertFalse("test" in db.collection_names())
db.test.drop()
db.drop_collection(db.test.doesnotexist)
def test_drop_collection(self):
db = Database(self.connection, "pymongo_test")
self.assertRaises(TypeError, db.drop_collection, 5)
self.assertRaises(TypeError, db.drop_collection, None)
db.test.save({"dummy": u"object"})
self.assertTrue("test" in db.collection_names())
db.drop_collection("test")
self.assertFalse("test" in db.collection_names())
db.test.save({"dummy": u"object"})
self.assertTrue("test" in db.collection_names())
db.drop_collection(u"test")
self.assertFalse("test" in db.collection_names())
db.test.save({"dummy": u"object"})
self.assertTrue("test" in db.collection_names())
db.drop_collection(db.test)
self.assertFalse("test" in db.collection_names())
db.test.save({"dummy": u"object"})
self.assertTrue("test" in db.collection_names())
db.test.drop()
self.assertFalse("test" in db.collection_names())
db.test.drop()
db.drop_collection(db.test.doesnotexist)
def test_repr(self):
self.assertEqual(repr(Database(self.connection, "pymongo_test")),
"Database(%r, u'pymongo_test')" % self.connection)
def test_drop_collection(self):
db = Database(self.connection, "pymongo_test")
self.assertRaises(TypeError, db.drop_collection, 5)
self.assertRaises(TypeError, db.drop_collection, None)
db.test.save({"dummy": u"object"})
self.assert_("test" in db.collection_names())
db.drop_collection("test")
self.assertFalse("test" in db.collection_names())
db.test.save({"dummy": u"object"})
self.assert_("test" in db.collection_names())
db.drop_collection(u"test")
self.assertFalse("test" in db.collection_names())
db.test.save({"dummy": u"object"})
self.assert_("test" in db.collection_names())
db.drop_collection(db.test)
self.assertFalse("test" in db.collection_names())
db.test.save({"dummy": u"object"})
self.assert_("test" in db.collection_names())
db.test.drop()
self.assertFalse("test" in db.collection_names())
db.test.drop()
db.drop_collection(db.test.doesnotexist)
def test_repr(self):
self.assertEqual(
repr(Database(self.connection, "pymongo_test")),
"Database(%r, %s)" % (self.connection, repr(u"pymongo_test")))
def setUp(self):
self.db = Database(get_connection(), "pymongo_test")
def test_get_coll(self):
db = Database(self.connection, "pymongo_test")
self.assertEqual(db.test, db["test"])
self.assertEqual(db.test, Collection(db, "test"))
self.assertNotEqual(db.test, Collection(db, "mike"))
self.assertEqual(db.test.mike, db["test.mike"])
def send(body):
with pika.BlockingConnection(pika.URLParameters(URL)) as connection:
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.basic_publish(exchange='', routing_key=QUEUE, body=body)
# print(f'[x] Sent {body}')
if __name__ == '__main__':
""""""
mongo_url = 'mongodb://192.168.12.102:27017'
mongo_db = 'company'
mongo_col = 'items'
client = MongoClient(mongo_url)
database = Database(client, mongo_db)
collection = Collection(database, mongo_col)
cursor = collection.find(limit=10000000)
import pytz
flag = 0
# for i in cursor:
# flag += 1
# if flag % 10000 == 0:
# print(f'flag: {flag}')
# send(json.dumps(i, cls=CustomsJsonEncoder))
#
# for i in range(0, 5):
# body = {
# "a": i,
# "b": "2",
class TestCursor(unittest.TestCase):
def setUp(self):
self.db = Database(get_connection(), "pymongo_test")
def test_explain(self):
a = self.db.test.find()
b = a.explain()
for _ in a:
break
c = a.explain()
del b["millis"]
b.pop("oldPlan", None)
del c["millis"]
c.pop("oldPlan", None)
self.assertEqual(b, c)
self.assert_("cursor" in b)
def test_hint(self):
db = self.db
self.assertRaises(TypeError, db.test.find().hint, 5.5)
db.test.remove({})
db.test.drop_indexes()
for i in range(100):
db.test.insert({"num": i, "foo": i})
self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}).hint([("num", ASCENDING)]).explain)
self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}).hint([("foo", ASCENDING)]).explain)
index = db.test.create_index("num")
spec = [("num", ASCENDING)]
self.assertEqual(db.test.find({}).explain()["cursor"], "BasicCursor")
self.assertEqual(db.test.find({}).hint(spec).explain()["cursor"], "BtreeCursor %s" % index)
self.assertEqual(db.test.find({}).hint(spec).hint(None).explain()["cursor"], "BasicCursor")
self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}).hint([("foo", ASCENDING)]).explain)
a = db.test.find({"num": 17})
a.hint(spec)
for _ in a:
break
self.assertRaises(InvalidOperation, a.hint, spec)
self.assertRaises(TypeError, db.test.find().hint, index)
# This is deprecated - test that a warning is actually raised
def test_slave_okay(self):
db = self.db
db.drop_collection("test")
db.test.save({"x": 1})
warnings.simplefilter("error")
self.assertEqual(1, db.test.find().next()["x"])
self.assertRaises(DeprecationWarning, db.test.find, slave_okay=True)
self.assertRaises(DeprecationWarning, db.test.find, slave_okay=False)
warnings.simplefilter("default")
def test_limit(self):
db = self.db
self.assertRaises(TypeError, db.test.find().limit, None)
self.assertRaises(TypeError, db.test.find().limit, "hello")
self.assertRaises(TypeError, db.test.find().limit, 5.5)
db.test.remove({})
for i in range(100):
db.test.save({"x": i})
count = 0
for _ in db.test.find():
count += 1
self.assertEqual(count, 100)
count = 0
for _ in db.test.find().limit(20):
count += 1
self.assertEqual(count, 20)
count = 0
for _ in db.test.find().limit(99):
count += 1
self.assertEqual(count, 99)
count = 0
for _ in db.test.find().limit(1):
count += 1
self.assertEqual(count, 1)
count = 0
for _ in db.test.find().limit(0):
count += 1
self.assertEqual(count, 100)
count = 0
for _ in db.test.find().limit(0).limit(50).limit(10):
count += 1
self.assertEqual(count, 10)
a = db.test.find()
a.limit(10)
for _ in a:
break
self.assertRaises(InvalidOperation, a.limit, 5)
def test_skip(self):
db = self.db
self.assertRaises(TypeError, db.test.find().skip, None)
self.assertRaises(TypeError, db.test.find().skip, "hello")
self.assertRaises(TypeError, db.test.find().skip, 5.5)
db.drop_collection("test")
for i in range(100):
db.test.save({"x": i})
for i in db.test.find():
self.assertEqual(i["x"], 0)
break
for i in db.test.find().skip(20):
self.assertEqual(i["x"], 20)
break
for i in db.test.find().skip(99):
self.assertEqual(i["x"], 99)
break
for i in db.test.find().skip(1):
self.assertEqual(i["x"], 1)
break
for i in db.test.find().skip(0):
self.assertEqual(i["x"], 0)
break
for i in db.test.find().skip(0).skip(50).skip(10):
self.assertEqual(i["x"], 10)
break
for i in db.test.find().skip(1000):
self.fail()
a = db.test.find()
a.skip(10)
for _ in a:
break
self.assertRaises(InvalidOperation, a.skip, 5)
def test_sort(self):
db = self.db
self.assertRaises(TypeError, db.test.find().sort, 5)
self.assertRaises(ValueError, db.test.find().sort, [])
self.assertRaises(TypeError, db.test.find().sort, [], ASCENDING)
self.assertRaises(TypeError, db.test.find().sort, [("hello", DESCENDING)], DESCENDING)
self.assertRaises(TypeError, db.test.find().sort, "hello", "world")
db.test.remove({})
unsort = range(10)
random.shuffle(unsort)
for i in unsort:
db.test.save({"x": i})
asc = [i["x"] for i in db.test.find().sort("x", ASCENDING)]
self.assertEqual(asc, range(10))
asc = [i["x"] for i in db.test.find().sort("x")]
self.assertEqual(asc, range(10))
asc = [i["x"] for i in db.test.find().sort([("x", ASCENDING)])]
self.assertEqual(asc, range(10))
expect = range(10)
expect.reverse()
desc = [i["x"] for i in db.test.find().sort("x", DESCENDING)]
self.assertEqual(desc, expect)
desc = [i["x"] for i in db.test.find().sort([("x", DESCENDING)])]
self.assertEqual(desc, expect)
desc = [i["x"] for i in db.test.find().sort("x", ASCENDING).sort("x", DESCENDING)]
self.assertEqual(desc, expect)
expected = [(1, 5), (2, 5), (0, 3), (7, 3), (9, 2), (2, 1), (3, 1)]
shuffled = list(expected)
random.shuffle(shuffled)
db.test.remove({})
for (a, b) in shuffled:
db.test.save({"a": a, "b": b})
result = [(i["a"], i["b"]) for i in db.test.find().sort([("b", DESCENDING), ("a", ASCENDING)])]
self.assertEqual(result, expected)
a = db.test.find()
a.sort("x", ASCENDING)
for _ in a:
break
self.assertRaises(InvalidOperation, a.sort, "x", ASCENDING)
def test_count(self):
db = self.db
db.test.remove({})
self.assertEqual(0, db.test.find().count())
for i in range(10):
db.test.save({"x": i})
self.assertEqual(10, db.test.find().count())
self.assert_(isinstance(db.test.find().count(), types.IntType))
self.assertEqual(10, db.test.find().limit(5).count())
self.assertEqual(10, db.test.find().skip(5).count())
self.assertEqual(1, db.test.find({"x": 1}).count())
self.assertEqual(5, db.test.find({"x": {"$lt": 5}}).count())
a = db.test.find()
b = a.count()
for _ in a:
break
self.assertEqual(b, a.count())
self.assertEqual(0, db.test.acollectionthatdoesntexist.find().count())
def test_where(self):
db = self.db
db.test.remove({})
a = db.test.find()
self.assertRaises(TypeError, a.where, 5)
self.assertRaises(TypeError, a.where, None)
self.assertRaises(TypeError, a.where, {})
for i in range(10):
db.test.save({"x": i})
self.assertEqual(3, len(list(db.test.find().where("this.x < 3"))))
self.assertEqual(3, len(list(db.test.find().where(Code("this.x < 3")))))
self.assertEqual(3, len(list(db.test.find().where(Code("this.x < i", {"i": 3})))))
self.assertEqual(10, len(list(db.test.find())))
self.assertEqual(3, db.test.find().where("this.x < 3").count())
self.assertEqual(10, db.test.find().count())
self.assertEqual(3, db.test.find().where(u"this.x < 3").count())
self.assertEqual([0, 1, 2], [a["x"] for a in db.test.find().where("this.x < 3")])
self.assertEqual([], [a["x"] for a in db.test.find({"x": 5}).where("this.x < 3")])
self.assertEqual([5], [a["x"] for a in db.test.find({"x": 5}).where("this.x > 3")])
cursor = db.test.find().where("this.x < 3").where("this.x > 7")
self.assertEqual([8, 9], [a["x"] for a in cursor])
a = db.test.find()
b = a.where("this.x > 3")
for _ in a:
break
self.assertRaises(InvalidOperation, a.where, "this.x < 3")
def test_kill_cursors(self):
db = self.db
db.drop_collection("test")
client_cursors = db._command({"cursorInfo": 1})["clientCursors_size"]
by_location = db._command({"cursorInfo": 1})["byLocation_size"]
for i in range(10000):
db.test.insert({"i": i})
self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])
for _ in range(10):
db.test.find_one()
self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])
for _ in range(10):
for x in db.test.find():
break
self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])
a = db.test.find()
for x in a:
break
self.assertNotEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
self.assertNotEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])
del a
self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])
a = db.test.find().limit(10)
for x in a:
break
self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])
def test_rewind(self):
self.db.test.save({"x": 1})
self.db.test.save({"x": 2})
self.db.test.save({"x": 3})
cursor = self.db.test.find().limit(2)
count = 0
for _ in cursor:
count += 1
self.assertEqual(2, count)
count = 0
for _ in cursor:
count += 1
self.assertEqual(0, count)
cursor.rewind()
count = 0
for _ in cursor:
count += 1
self.assertEqual(2, count)
cursor.rewind()
count = 0
for _ in cursor:
break
cursor.rewind()
for _ in cursor:
count += 1
self.assertEqual(2, count)
self.assertEqual(cursor, cursor.rewind())
def test_clone(self):
self.db.test.save({"x": 1})
self.db.test.save({"x": 2})
self.db.test.save({"x": 3})
cursor = self.db.test.find().limit(2)
count = 0
for _ in cursor:
count += 1
self.assertEqual(2, count)
count = 0
for _ in cursor:
count += 1
self.assertEqual(0, count)
cursor = cursor.clone()
cursor2 = cursor.clone()
count = 0
for _ in cursor:
count += 1
self.assertEqual(2, count)
for _ in cursor2:
count += 1
self.assertEqual(4, count)
cursor.rewind()
count = 0
for _ in cursor:
break
cursor = cursor.clone()
for _ in cursor:
count += 1
self.assertEqual(2, count)
self.assertNotEqual(cursor, cursor.clone())
def test_count_with_fields(self):
self.db.test.remove({})
self.db.test.save({"x": 1})
if not version.at_least(self.db.connection(), (1, 1, 3, -1)):
for _ in self.db.test.find({}, ["a"]):
self.fail()
self.assertEqual(0, self.db.test.find({}, ["a"]).count())
else:
self.assertEqual(1, self.db.test.find({}, ["a"]).count())
def test_bad_getitem(self):
self.assertRaises(TypeError, lambda x: self.db.test.find()[x], "hello")
self.assertRaises(TypeError, lambda x: self.db.test.find()[x], 5.5)
self.assertRaises(TypeError, lambda x: self.db.test.find()[x], None)
def test_getitem_slice_index(self):
self.db.drop_collection("test")
for i in range(100):
self.db.test.save({"i": i})
izip = itertools.izip
count = itertools.count
self.assertRaises(IndexError, lambda: self.db.test.find()[-1:])
self.assertRaises(IndexError, lambda: self.db.test.find()[1:2:2])
for a, b in izip(count(0), self.db.test.find()):
self.assertEqual(a, b["i"])
self.assertEqual(100, len(list(self.db.test.find()[0:])))
for a, b in izip(count(0), self.db.test.find()[0:]):
self.assertEqual(a, b["i"])
self.assertEqual(80, len(list(self.db.test.find()[20:])))
for a, b in izip(count(20), self.db.test.find()[20:]):
self.assertEqual(a, b["i"])
for a, b in izip(count(99), self.db.test.find()[99:]):
self.assertEqual(a, b["i"])
for i in self.db.test.find()[1000:]:
self.fail()
self.assertEqual(5, len(list(self.db.test.find()[20:25])))
self.assertEqual(5, len(list(self.db.test.find()[20L:25L])))
for a, b in izip(count(20), self.db.test.find()[20:25]):
self.assertEqual(a, b["i"])
self.assertEqual(80, len(list(self.db.test.find()[40:45][20:])))
for a, b in izip(count(20), self.db.test.find()[40:45][20:]):
self.assertEqual(a, b["i"])
self.assertEqual(80, len(list(self.db.test.find()[40:45].limit(0).skip(20))))
for a, b in izip(count(20), self.db.test.find()[40:45].limit(0).skip(20)):
self.assertEqual(a, b["i"])
self.assertEqual(80, len(list(self.db.test.find().limit(10).skip(40)[20:])))
for a, b in izip(count(20), self.db.test.find().limit(10).skip(40)[20:]):
self.assertEqual(a, b["i"])
self.assertEqual(1, len(list(self.db.test.find()[:1])))
self.assertEqual(5, len(list(self.db.test.find()[:5])))
self.assertEqual(1, len(list(self.db.test.find()[99:100])))
self.assertEqual(1, len(list(self.db.test.find()[99:1000])))
self.assertRaises(IndexError, lambda: self.db.test.find()[10:8])
self.assertRaises(IndexError, lambda: self.db.test.find()[10:10])
self.assertRaises(IndexError, lambda: self.db.test.find()[:0])
def test_getitem_numeric_index(self):
self.db.drop_collection("test")
for i in range(100):
self.db.test.save({"i": i})
self.assertEqual(0, self.db.test.find()[0]["i"])
self.assertEqual(50, self.db.test.find()[50]["i"])
self.assertEqual(50, self.db.test.find().skip(50)[0]["i"])
self.assertEqual(50, self.db.test.find().skip(49)[1]["i"])
self.assertEqual(50, self.db.test.find()[50L]["i"])
self.assertEqual(99, self.db.test.find()[99]["i"])
self.assertRaises(IndexError, lambda x: self.db.test.find()[x], -1)
self.assertRaises(IndexError, lambda x: self.db.test.find()[x], 100)
self.assertRaises(IndexError, lambda x: self.db.test.find().skip(50)[x], 50)
def test_count_with_limit_and_skip(self):
if not version.at_least(self.db.connection(), (1, 1, 4, -1)):
raise SkipTest()
def check_len(cursor, length):
self.assertEqual(len(list(cursor)), cursor.count(True))
self.assertEqual(length, cursor.count(True))
self.db.drop_collection("test")
for i in range(100):
self.db.test.save({"i": i})
check_len(self.db.test.find(), 100)
check_len(self.db.test.find().limit(10), 10)
check_len(self.db.test.find().limit(110), 100)
check_len(self.db.test.find().skip(10), 90)
check_len(self.db.test.find().skip(110), 0)
check_len(self.db.test.find().limit(10).skip(10), 10)
check_len(self.db.test.find()[10:20], 10)
check_len(self.db.test.find().limit(10).skip(95), 5)
check_len(self.db.test.find()[95:105], 5)
def test_len(self):
self.assertRaises(TypeError, len, self.db.test.find())
def test_properties(self):
self.assertEqual(self.db.test, self.db.test.find().collection)
def set_coll():
self.db.test.find().collection = "hello"
self.assertRaises(AttributeError, set_coll)
def test_tailable(self):
db = self.db
db.drop_collection("test")
cursor = db.test.find(tailable=True)
db.test.insert({"x": 1})
count = 0
for doc in cursor:
count += 1
self.assertEqual(1, doc["x"])
self.assertEqual(1, count)
db.test.insert({"x": 2})
count = 0
for doc in cursor:
count += 1
self.assertEqual(2, doc["x"])
self.assertEqual(1, count)
db.test.insert({"x": 3})
count = 0
for doc in cursor:
count += 1
self.assertEqual(3, doc["x"])
self.assertEqual(1, count)
self.assertEqual(3, db.test.count())
def test_distinct(self):
if not version.at_least(self.db.connection(), (1, 1, 3, 1)):
raise SkipTest()
self.db.drop_collection("test")
self.db.test.save({"a": 1})
self.db.test.save({"a": 2})
self.db.test.save({"a": 2})
self.db.test.save({"a": 2})
self.db.test.save({"a": 3})
distinct = self.db.test.find({"a": {"$lt": 3}}).distinct("a")
distinct.sort()
self.assertEqual([1, 2], distinct)
self.db.drop_collection("test")
self.db.test.save({"a": {"b": "a"}, "c": 12})
self.db.test.save({"a": {"b": "b"}, "c": 8})
self.db.test.save({"a": {"b": "c"}, "c": 12})
self.db.test.save({"a": {"b": "c"}, "c": 8})
distinct = self.db.test.find({"c": 8}).distinct("a.b")
distinct.sort()
self.assertEqual(["b", "c"], distinct)
def test_create_collection(self):
db = Database(self.connection, "pymongo_test")
db.test.insert({"hello": "world"})
self.assertRaises(CollectionInvalid, db.create_collection, "test")
db.drop_collection("test")
self.assertRaises(TypeError, db.create_collection, 5)
self.assertRaises(TypeError, db.create_collection, None)
self.assertRaises(InvalidName, db.create_collection, "coll..ection")
test = db.create_collection("test")
test.save({"hello": u"world"})
self.assertEqual(db.test.find_one()["hello"], "world")
self.assertTrue(u"test" in db.collection_names())
db.drop_collection("test.foo")
db.create_collection("test.foo")
self.assertTrue(u"test.foo" in db.collection_names())
self.assertEqual(db.test.foo.options(), {})
self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
def __init__(self, host, db_name='test', collection_name='fs'):
self.client = MongoClient(host)
self.db = Database(self.client, db_name)
self.fs = GridFS(self.db, collection_name)
if result.matched_count == 0:
self.log.warning("Experiement does not exist. Created: " +
experiment_id)
else:
self.log.warning("FAILED to update state for experiment: " +
experiment_id)
if __name__ == "__main__":
import yaml
import logging.config
with open("logging.yaml", 'r') as logconfig:
cfg = yaml.safe_load(logconfig)
logging.config.dictConfig(cfg)
logger = logging.getLogger('mainLogger')
url = "10.42.117.13"
port = 27017
dbName = "beaglemlDB"
docName = "duymmyMetrics"
client = MongoClient(url, port)
db = Database(client, dbName)
mw = mongoWriter(db, logger, docName)
metric = {"time": 1}
mw.write_metrics(metric)
def getMongoCol():
uri = "mongodb://%s" % ('localhost:27017')
client = MongoClient(uri)
db = Database(client, 'test')
col = db.get_collection('images')
return col
# -*- encoding: utf-8 -*- ''' Created on 2014-11-7 @author: [email protected] ''' import pymongo from pymongo.database import Database from pymongo.mongo_client import MongoClient from pymongo.collection import Collection from gridfs import GridFS if __name__ == '__main__': con = pymongo.Connection('172.16.10.170', 27017) # con = MongoClient(host='172.16.10.170', port=27017, max_pool_size=200) db = Database(con, 'blog') db.authenticate('admin', 'admin') coll = Collection(db, 'foo') coll.insert({'a':1, 'b':2, 'c':3}) # coll.update({'a':1}, {'$set':{'b':4}}, multi=True) print [x for x in coll.find()] # MongoClient() # print user.find({'id':1}).count() pass