Beispiel #1
0
def store_feeds(feeds, collection):
    'store the feeds in mongodb '
    from pymongo.errors import CollectionInvalid
    from pymongo.collection import Collection
    from pymongo.connection import Connection
    con = Connection(DB)
    from pymongo.database import Database
    db = Database(con, 'articles')

    col = None
    try:
        col = db.create_collection(collection)
    except CollectionInvalid as e:
        col = Collection(db, collection)

    for feed in feeds:
        if 'title' in feed:
            cursor = col.find({'title':'%s' % feed['title']})
            if cursor.count() > 0:
                continue
        elif 'source' in feed:
            cursor = col.find({'source':'%s' % feed['source']})
            if cursor.count() > 0:
                continue
        col.save(feed)
    def test_create_collection(self):
        db = Database(self.client, "pymongo_test")

        db.test.insert({"hello": "world"})
        self.assertRaises(CollectionInvalid, db.create_collection, "test")

        db.drop_collection("test")

        self.assertRaises(TypeError, db.create_collection, 5)
        self.assertRaises(TypeError, db.create_collection, None)
        self.assertRaises(InvalidName, db.create_collection, "coll..ection")

        test = db.create_collection("test")
        test.save({"hello": u"world"})
        self.assertEqual(db.test.find_one()["hello"], "world")
        self.assertTrue(u"test" in db.collection_names())

        db.drop_collection("test.foo")
        db.create_collection("test.foo")
        self.assertTrue(u"test.foo" in db.collection_names())
        expected = {}
        if version.at_least(self.client, (2, 7, 0)):
            # usePowerOf2Sizes server default
            expected["flags"] = 1
        result = db.test.foo.options()
        # mongos 2.2.x adds an $auth field when auth is enabled.
        result.pop("$auth", None)
        self.assertEqual(result, expected)
        self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
 def binary_contents_test(self):
     db = Database(self._get_connection(), "pymongo_test")
     test = db.create_collection("test_binary")
     import os
     import bson
     obj = os.urandom(1024)
     test.save({"hello": bson.Binary(obj)})
     db.drop_collection("test_binary")
    def test_collection_names(self):
        db = Database(self.connection, "pymongo_test")
        db.test.save({"dummy": u"object"})
        db.test.mike.save({"dummy": u"object"})

        colls = db.collection_names()
        self.assert_("test" in colls)
        self.assert_("test.mike" in colls)
        for coll in colls:
            self.assert_("$" not in coll)
Beispiel #5
0
 def __init__(self, config):
     database = config['mongo_db_name']
     if 'mongo_login' in config:
         passwordfile = config['mongo_login']
     else:
         passwordfile = None
     connection = MongoDB.connect(config)
     Database.__init__(self, connection, database)
     if passwordfile is not None and exists(passwordfile):
         password = yaml.load(open(passwordfile, 'rU'))
         self.authenticate(password['mongo_user'], password['mongo_password'])
    def test_collection_names(self):
        db = Database(self.client, "pymongo_test")
        db.test.save({"dummy": u"object"})
        db.test.mike.save({"dummy": u"object"})

        colls = db.collection_names()
        self.assertTrue("test" in colls)
        self.assertTrue("test.mike" in colls)
        for coll in colls:
            self.assertTrue("$" not in coll)

        colls_without_systems = db.collection_names(False)
        for coll in colls_without_systems:
            self.assertTrue(not coll.startswith("system."))
Beispiel #7
0
def get_db(is_local_deployed=False):
    if is_local_deployed:
        #for local launch
        connection = Connection()
        db = connection.wiki
        return db
    else:
        # for dotcloud launch
        mongodb_info = json.load(file("/home/dotcloud/environment.json"))
        print mongodb_info
        connection = Connection(mongodb_info["DOTCLOUD_DATA_MONGODB_HOST"], int(mongodb_info["DOTCLOUD_DATA_MONGODB_PORT"]))
        database = Database(connection, "wiki")
        database.authenticate("xinz", "hellopy")
        db = connection.wiki
        return db
    def test4(self):
        db = Database(self._get_connection(), "pymongo_test")
        test = db.create_collection("test_4")
        try:
            for i in range(5):
                name = "test %d" % (i)
                test.save({ "user_id": i, "name": name, "group_id" : i % 10, "posts": i % 20})

            test.create_index("user_id")

            for i in xrange(6):
                for r in test.find( { "group_id": random.randint(0,10) } ):
                    print "Found: %s " % (r)

        finally:
            db.drop_collection("test_4")
Beispiel #9
0
class MongoopTrigger(BaseTrigger):
    def __init__(self, *args, **kwargs):
        super(MongoopTrigger, self).__init__(*args, **kwargs)

        database = self.params.get('database', 'mongoop')
        collection = self.params.get('collection', 'history')

        self.db = Database(self.mongoop.conn, database)
        self.collection = self.db.get_collection(collection)
        self.collection.create_index([('opid', DESCENDING)],
                                     unique=True,
                                     background=True)

    def op_nok(self, operations):
        try:
            if operations:
                self.collection.insert_many(operations)
        except Exception as e:
            logging.error('unable to bulk operations :: {} :: {}'.format(
                self.name, e))
            return False
        else:
            logging.info('run :: {} :: bulk insert {} operations'.format(
                self.name, len(operations)))
            return True
Beispiel #10
0
    def __init__(self):
        db_server_ip = conf("config.db_server_ip")
        db_server_port = conf("config.db_server_port")
        self.__connection = Connection(host=db_server_ip, port=db_server_port)

        db_name = conf("config.db_name")
        self.__db = Database(connection= self.__connection, name=db_name)
Beispiel #11
0
def borrarindicesevitardups():
    # Conexion a local
    client = MongoClient()
    
    try:
        # The ismaster command is cheap and does not require auth.
        client.admin.command('ismaster')
        db = Database(client=client,
                      name=ConfiguracionGlobal.MONGODB)
    except ConnectionFailure:
        print("Server not available")
    # Limpiado tablas e indices con pymongo
    for collname in db.collection_names():
        coll = Collection(db, name=collname)
        coll.drop_indexes()
    
    client.close()
Beispiel #12
0
def connect(conf, prod):
    log.info('setting up mongo connection')

    url = conf.get('mongo', 'url')
    port = conf.getint('mongo', 'port')
    db = conf.get('mongo', 'db')

    conn = Connection(url, port)
    db = Database(conn, db)

    if (prod):
        log.info('authenticating mongo connection')
        username = conf.get('mongo', 'username')
        pwd = conf.get('mongo', 'password')

        db.authenticate(username, pwd)

    return db
Beispiel #13
0
    def __init__(self, *args, **kwargs):
        super(MongoopTrigger, self).__init__(*args, **kwargs)

        database = self.params.get('database', 'mongoop')
        collection = self.params.get('collection', 'history')

        self.db = Database(self.mongoop.conn, database)
        self.collection = self.db.get_collection(collection)
        self.collection.create_index([('opid', DESCENDING)], unique=True, background=True)
Beispiel #14
0
 def __init__(self, conn_string, database, trace=False):
     if not conn_string.startswith('mongodb://'):
         conn_string = 'mongodb://' + conn_string
     self.connection = Connection(conn_string)
     self.database = Database(self.connection, database)
     self.trace = trace
     self.collections = []
     self.obj_infos = set()
     self._cache = {}
Beispiel #15
0
def store_feeds(feeds, collection):
    'store the feeds in mongodb '
    from pymongo.errors import CollectionInvalid
    from pymongo.collection import Collection
    from pymongo.connection import Connection
    con = Connection(DB)
    from pymongo.database import Database
    db = Database(con, 'queries')

    col = None
    try:
        col = db.create_collection(collection)
    except CollectionInvalid as e:
        col = Collection(db, collection)

    for feed in feeds:
        existed = col.find_one({'query':feed}, {'$exists':'true'})
        if not existed:
            col.save({'query':feed})
Beispiel #16
0
 def conn(self, line):
     """Conenct to mongo database in ipython shell.
     Examples::
         %mongo_connect
         %mongo_connect mongodb://hostname:port
     """
     if line:
         uri = line
     else:
         uri = 'mongodb://127.0.0.1:27017'
     self._conn = MongoClient(uri)
     # add db and collection property to object for autocomplete.
     for db in self._conn.database_names():
         setattr(self._conn, db, Database(self._conn, db))
         _db = Database(self._conn, db)
         for collection in _db.collection_names():
             # [TODO] change eval to other!!
             setattr(eval('self._conn.'+db), collection,
                     Collection(_db, collection))
     return self._conn
    def test_get_collection(self):
        db = Database(self.client, "pymongo_test")
        codec_options = CodecOptions(
            tz_aware=True, uuid_representation=JAVA_LEGACY)
        write_concern = WriteConcern(w=2, j=True)
        coll = db.get_collection(
            'foo', codec_options, ReadPreference.SECONDARY, write_concern)
        self.assertEqual('foo', coll.name)
        self.assertEqual(codec_options, coll.codec_options)
        self.assertEqual(JAVA_LEGACY, coll.uuid_subtype)
        self.assertEqual(ReadPreference.SECONDARY, coll.read_preference)
        self.assertEqual(write_concern.document, coll.write_concern)

        pref = Secondary([{"dc": "sf"}])
        coll = db.get_collection('foo', read_preference=pref)
        self.assertEqual(pref.mode, coll.read_preference)
        self.assertEqual(pref.tag_sets, coll.tag_sets)
        self.assertEqual(db.codec_options, coll.codec_options)
        self.assertEqual(db.uuid_subtype, coll.uuid_subtype)
        self.assertEqual(db.write_concern, coll.write_concern)
    def test_create_collection(self):
        db = Database(self.client, "pymongo_test")

        db.test.insert_one({"hello": "world"})
        self.assertRaises(CollectionInvalid, db.create_collection, "test")

        db.drop_collection("test")

        self.assertRaises(TypeError, db.create_collection, 5)
        self.assertRaises(TypeError, db.create_collection, None)
        self.assertRaises(InvalidName, db.create_collection, "coll..ection")

        test = db.create_collection("test")
        self.assertTrue(u("test") in db.collection_names())
        test.insert_one({"hello": u("world")})
        self.assertEqual(db.test.find_one()["hello"], "world")

        db.drop_collection("test.foo")
        db.create_collection("test.foo")
        self.assertTrue(u("test.foo") in db.collection_names())
        self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
Beispiel #19
0
    def init_connection(self):
        if hasattr(self, 'app'):
            config = self.app.config
        else:
            config = self._default_config

        connection = Connection(
            host=config.get('MONGODB_HOST'),
            port=config.get('MONGODB_PORT'),
            slave_okay=config.get('MONGODB_SLAVE_OKAY')
        )

        database = Database(connection, config.get('MONGODB_DATABASE'))

        if config.get('MONGODB_USERNAME') is not None:
            database.authenticate(
                config.get('MONGODB_USERNAME'),
                config.get('MONGODB_PASSWORD')
            )

        return connection, database
def load_feature_vectors_and_classes(db_name):
    """
    :param db_name: name of database to use
    :return:
        - list of feature vectors
        - dictionary where the keys are the class labels and the values are dictionaries of the form
        {start: <integer>, end: <integer>}
    """
    print 'Loading feature vectors and target classes...'
    db = Database(MongoClient(), db_name)
    collection_names = db.collection_names()
    if not ('naive_bayes' in collection_names and 'feature_vectors' in collection_names):
        print 'Database missing collections needed to train classifier on.'
        exit()

    target_classes = Collection(db, 'naive_bayes').find_one({'type': 'classes'})
    if 'classes' not in target_classes:
        raise KeyError("'target_classes' must contain a 'classes' key.")

    feature_vectors = list(Collection(db, 'feature_vectors').find())
    return feature_vectors, target_classes['classes']
    def test_create_collection(self):
        db = Database(self.connection, "pymongo_test")

        db.test.insert({"hello": "world"})
        self.assertRaises(CollectionInvalid, db.create_collection, "test")

        db.drop_collection("test")

        self.assertRaises(TypeError, db.create_collection, 5)
        self.assertRaises(TypeError, db.create_collection, None)
        self.assertRaises(InvalidName, db.create_collection, "coll..ection")

        test = db.create_collection("test")
        test.save({"hello": u"world"})
        self.assertEqual(db.test.find_one()["hello"], "world")
        self.assert_(u"test" in db.collection_names())

        db.drop_collection("test.foo")
        db.create_collection("test.foo")
        self.assert_(u"test.foo" in db.collection_names())
        self.assertEqual(db.test.foo.options(), {})
        self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
    def test2(self):
        db = Database(self._get_connection(), "pymongo_test")
        test = db.create_collection("test_2")
        try:
            for i in range(100):
                name = "test %d" % (i)
                ret = test.save({"name": name, "group_id" : i % 3, "posts": i % 20})
                print "Save Ret: %s" % (ret)

            ret = test.update({"posts": 10}, {"$set": {"posts": 100}}, multi=True, safe=True)
            #ret = test.update({"posts": 10}, {"$set": {"posts": 100}}, multi=True)
            print "Update Ret: %s" % (ret)
            test.update({"name": "test 2"}, {"$set": {"posts": 200}})
            test.create_index("posts")
            test.ensure_index("posts")

            for r in test.find({"posts":100}):
                print "Found: %s" % (r,)

            ret = test.remove({"posts": 1}, safe=True)
            print "Remove Ret: %s" % (ret)

            groups = test.group(
                key={"group_id":1},
                condition=None,
                initial={"post_sum":0},
                reduce="function(obj,prev) {prev.post_sum++;}"
            )
            for g in groups:
                print "Group: %s" % (g,)

            for d in test.distinct('posts'):
                print "Distinct: %s" % (d,)

            if 'reindex' in dir(test):
                test.reindex()
            test.drop_indexes()
        finally:
            db.drop_collection("test_2")
    def test_collection_names(self):
        db = Database(self.client, "pymongo_test")
        db.test.insert_one({"dummy": u("object")})
        db.test.mike.insert_one({"dummy": u("object")})

        colls = db.collection_names()
        self.assertTrue("test" in colls)
        self.assertTrue("test.mike" in colls)
        for coll in colls:
            self.assertTrue("$" not in coll)

        colls_without_systems = db.collection_names(False)
        for coll in colls_without_systems:
            self.assertTrue(not coll.startswith("system."))

        # Force more than one batch.
        db = self.client.many_collections
        for i in range(101):
            db["coll" + str(i)].insert_one({})
        # No Error
        try:
            db.collection_names()
        finally:
            self.client.drop_database("many_collections")
Beispiel #24
0
    def test_get_db(self):
        client = MongoClient(host, port)

        def make_db(base, name):
            return base[name]

        self.assertRaises(InvalidName, make_db, client, "")
        self.assertRaises(InvalidName, make_db, client, "te$t")
        self.assertRaises(InvalidName, make_db, client, "te.t")
        self.assertRaises(InvalidName, make_db, client, "te\\t")
        self.assertRaises(InvalidName, make_db, client, "te/t")
        self.assertRaises(InvalidName, make_db, client, "te st")

        self.assertTrue(isinstance(client.test, Database))
        self.assertEqual(client.test, client["test"])
        self.assertEqual(client.test, Database(client, "test"))
Beispiel #25
0
    def test_get_db(self):
        connection = Connection(self.host, self.port)

        def make_db(base, name):
            return base[name]

        self.assertRaises(InvalidName, make_db, connection, "")
        self.assertRaises(InvalidName, make_db, connection, "te$t")
        self.assertRaises(InvalidName, make_db, connection, "te.t")
        self.assertRaises(InvalidName, make_db, connection, "te\\t")
        self.assertRaises(InvalidName, make_db, connection, "te/t")
        self.assertRaises(InvalidName, make_db, connection, "te st")

        self.assertTrue(isinstance(connection.test, Database))
        self.assertEqual(connection.test, connection["test"])
        self.assertEqual(connection.test, Database(connection, "test"))
    def test_get_db(self):
        connection = self._get_connection()

        def make_db(base, name):
            return base[name]

        self.assertRaises(InvalidName, make_db, connection, "")
        self.assertRaises(InvalidName, make_db, connection, "te$t")
        self.assertRaises(InvalidName, make_db, connection, "te.t")
        self.assertRaises(InvalidName, make_db, connection, "te\\t")
        self.assertRaises(InvalidName, make_db, connection, "te/t")
        self.assertRaises(InvalidName, make_db, connection, "te st")

        self.assert_(isinstance(connection.test, Database))
        self.assertEqual(connection.test, connection["test"])
        self.assertEqual(connection.test, Database(connection, "test"))
        connection.close()
def dbncol(client, collection, database='test'):
    ''' Make a connection to the database and collection given in the arguments.

    :param client: a MongoClient instance
    :type client: pymongo.MongoClient
    :param database: the name of the database to be used. It must be a database name present at the client
    :type database: str
    :param collection: the database collection to be used.  It must be a collection name present in the database
    :type collection: str
    
    :return col: the collection to be used
    :type: pymongo.collection.Collection
    '''

    db = Database(client, database)
    col = Collection(db, collection)
    return col
Beispiel #28
0
    def resolve_object_ids(self, mongo_object):
        if mongo_object is None:
            return None

        if type(mongo_object) is "dict":
            result = { }
            for key in mongo_object:
                result[key] = self.resolve_object_ids(mongo_object[key]) if key is not "_id" else mongo_object[key]
            return result
        elif type(mongo_object) is "list":
            result = []
            for item in mongo_object:
                result.append(self.resolve_object_ids(item))
            return result
        elif type(mongo_object) is "bson.objectid.ObjectId":
            return self.get_single(mongo_object)
        elif type(mongo_object) is "bson.dbref.DBRef":
            return Database.dereference(mongo_object)
    def getTweetsTimeSeries(self, mongoDbCollection : Database, stockTicker, fromDate, toDate):
        raw_tweets = mongoDbCollection.aggregate([
            {"$match": {"stockTicker": stockTicker, "date": {"$gte": fromDate, "$lt": toDate}}},
            {"$group": { "_id": {"day": {"$dayOfMonth": "$date"}, "month": {"$month": "$date"}, "year": {"$year": "$date"}},
            "count": {"$sum": 1}}}])
        raw_tweets = list(raw_tweets)

        date_count_tweets = []
        # extract tweets to be in list of dictionaries of form date and tweets
        # TODO upgrade mongo DB to 3.6 to do a projection with $dateFromParts
        for tweet in raw_tweets:
            entry = {}
            entry['date'] = datetime(year=tweet["_id"]["year"], month=tweet["_id"]["month"], day=tweet["_id"]["day"])
            entry['tweets'] = tweet['count']
            date_count_tweets.append(entry)
        date_count_tweets = sorted(date_count_tweets, key=lambda k: k['date'])

        return date_count_tweets
Beispiel #30
0
    def _conn():
        if not nosqlDB.con:
            try:
                # config = ConfigParser.ConfigParser()
                # config.readfp(open(PackageLoader('conf','db.ini')))
                nosqlDB.initParas = globalConf.nosqldb
                nosqlDB.con = Connection(host=nosqlDB.initParas['host'],
                                         port=nosqlDB.initParas['port'],
                                         max_pool_size=100)
                # nosqlDB.con = MongoClient(host=nosqlDB.initParas['host'], port=nosqlDB.initParas['port'], max_pool_size=200)

                nosqlDB.db = Database(nosqlDB.con, nosqlDB.initParas['db'])
                if nosqlDB.initParas['isauth'] == '1':
                    nosqlDB.db.auth(nosqlDB.initParas['user'],
                                    nosqlDB.initParas['passwd'])
                nosqlDB.gfs = GridFS(nosqlDB.db, 'files')
            except BaseException, e:
                print e
def find_data(client, database, collection, filters={}):
    ''' Find the items in the specified database and collection using the filters.

    :param client: a MongoClient instance
    :type client: pymongo.MongoClient
    :param database: the name of the database to be used. It must be a database name present at the client
    :type database: str
    :param collection: the database collection to be used.  It must be a collection name present in the database
    :type collection: str
    :param filters: the parameters used for filtering the returned data. An empty filter parameter returns the full collection
    :type filters: dict
    
    :return: the result of the query
    :type: pymongo.cursor.CursorType
    '''

    db = Database(client, database)
    col = Collection(db, collection)
    return col.find(filters).batch_size(100)
Beispiel #32
0
 def get_collection(self,
                    collection_name,
                    database=settings.MONGO_DEFAULT_DATABASE):
     '''
     :param collection_name: collection name
     :param database: database name
     :return: Collection object of the specified collection name.
     '''
     try:
         mongo_client = self.get_mongo_client()
         database_instance = Database(mongo_client, database)
         collection = Collection(database=database_instance,
                                 name=collection_name)
         return collection
     except (PyMongoError, Exception) as e:
         logger.error(
             f"Error connecting to collection {collection_name}  error = {e}"
         )
         raise e
Beispiel #33
0
def parse(cell, config):
    parts = [part.strip() for part in cell.split(None, 1)]
    if not parts:
        return """[ERROR] please enter the db.collection and data
            %insert db.collection {data} or [{data}, {data}...]
            or
            %%insert db.collection
            {data} or [{data}, {data} ...]"""
    db, col = parts[0].split('.')
    if not db and not col:
        return """[ERROR] please enter the db and collection name both of all
            %insert db.collection {data or data list}
            or
            %%insert db.collection
            {data or data list}"""
    _db = Database(config._conn, db)
    _col = Collection(_db, col)
    data = parts[1].strip() if len(parts) > 1 else ''
    return {'collection': _col, 'data': data}
Beispiel #34
0
    def __init__(self, logger, **kwargs):
        self.logger = logger
        self.host_array = kwargs.get('hosts')
        self.replicaset = kwargs.get('replicaset')
        self.db_name = kwargs.get('database_name')

        if not self.replicaset:
            host_tokens = self.host_array[0].split(':')

            hostname = host_tokens[0]
            port = 27017
            if len(host_tokens) == 2:
                port = int(host_tokens[1])

            mdb_client = MongoClient(host=hostname, port=port)
            self.db = Database(mdb_client, self.db_name)
        else:
            raise Exception(
                'Connect-to-replicaset not yet implemented (but coming soon)')
def calculate_user_storage(db: Database, user_id: ObjectId):
    """
    Get storage that the user has used
    :param db: Database instance
    :param user_id: The user's objectId
    :return: Storage size, a list of files
    """
    medias_size = 0
    file_set = set()
    # Get all collections and map data by user id
    for col in db.list_collection_names():
        data_list = db[col].aggregate([{
            '$match': {
                'userId': user_id
            }
        }, {
            "$addFields": {
                "paths": {
                    '$reduce': {
                        'input': "$items.example.medias",
                        'initialValue': [],
                        'in': {
                            '$concatArrays': ["$$value", "$$this.src"]
                        }
                    }
                }
            }
        }, {
            '$project': {
                'paths': 1
            }
        }, {
            '$unwind': '$paths'
        }])
        # Map all files' addresses
        for path_obj in data_list:
            if 'paths' in path_obj and path_obj['paths']:
                fp = path_obj['paths'][1:]
                if os.path.exists(fp) and fp not in file_set:
                    # Build status for storage
                    medias_size += os.path.getsize(fp)
                    file_set.add(fp)
    return medias_size, file_set
Beispiel #36
0
 def __init__(
     self,
     model_class: Type[T],
     database: Database,
     col_name: str,
     indexes: Optional[list[Union[IndexModel, str]]] = None,
     wrap_object_str_id=True,
 ):
     codecs = CodecOptions(
         type_registry=TypeRegistry([c() for c in [DecimalCodec]]))
     self.collection = database.get_collection(col_name, codecs)
     if indexes:
         indexes = [
             parse_str_index_model(i) if isinstance(i, str) else i
             for i in indexes
         ]
         self.collection.create_indexes(indexes)
     self.model_class = model_class
     self.wrap_object_id = model_class.__fields__[
         "id"].type_ == ObjectIdStr and wrap_object_str_id
Beispiel #37
0
def test_update_collection_schema(mongo_handler: MongoHandler,
                                  evo_db: Database,
                                  temp_coll_name: str,
                                  scheme_and_data: Tuple[Dict[str, Any]]):

    valid_scheme, valid_data, invalid_data = scheme_and_data
    collection = evo_db.get_collection(temp_coll_name)

    result = mongo_handler.update_collection_schema(
        temp_coll_name, valid_scheme)
    assert result

    with pytest.raises(WriteError):
        collection.insert_one(invalid_data)

    doc_id = collection.insert_one(valid_data)
    assert doc_id is not None

    col_name = "imaginary"
    result = mongo_handler.update_collection_schema(col_name, valid_scheme)
    assert not result
Beispiel #38
0
def update_collections(db: Database, refresh: bool) -> None:
    """
    If covid or states collections don't exist in db, downloads data, creates collections, and inserts data
    :param refresh: If True, download current data and update database
    :param db: PyMongo Database in which to update collections
    """
    collections = db.list_collection_names()

    if COLL_COVID not in collections or refresh:
        if refresh:
            db[COLL_COVID].drop()
        covid_data: JSON = get_covid_data()
        add_collection(db, COLL_COVID, covid_data)
        fix_covid_data(db, COLL_COVID)

    if COLL_STATES not in collections or refresh:
        if refresh:
            db[COLL_STATES].drop()
        states_data: JSON = get_states_data()
        add_cumulative_properties(states_data)
        add_collection(db, COLL_STATES, states_data)
        fix_states_data(db, COLL_STATES)
Beispiel #39
0
    def __init__(self,
                 domain,
                 host="mongodb://localhost:27017",
                 db="rasa",
                 username=None,
                 password=None,
                 collection="conversations",
                 event_broker=None):
        from pymongo.database import Database
        from pymongo import MongoClient

        self.client = MongoClient(host,
                                  username=username,
                                  password=password,
                                  # delay connect until process forking is done
                                  connect=False)

        self.db = Database(self.client, db)
        self.collection = collection
        super(MongoTrackerStore, self).__init__(domain, event_broker)

        self._ensure_indices()
Beispiel #40
0
def encode_model(obj, recursive=False):
    if obj is None:
        return obj
    if isinstance(obj, (Document, EmbeddedDocument)):
        out = dict(obj._data)
        for k, v in out.items():
            if isinstance(v, ObjectId):
                if k is None:
                    del(out[k])
                else:
                    # Unlikely that we'll hit this since ObjectId is always NULL key
                    out[k] = str(v)
            else:
                out[k] = encode_model(v)
    elif isinstance(obj, ModuleType):
        out = None
    elif isinstance(obj, groupby):
        out = [(gb, list(l)) for gb, l in obj]
    elif isinstance(obj, (list)):
        out = [encode_model(item) for item in obj]

    elif isinstance(obj, (dict)):
        out = dict([(k, encode_model(v)) for (k, v) in obj.items()])

    elif hasattr(obj, 'isoformat'):
        out = obj.isoformat()
    elif isinstance(obj, ObjectId):
        return None
    elif isinstance(obj, (str, unicode, int)):
        out = obj
    elif isinstance(obj, float):
        out = str(obj)
    elif isinstance(obj, DBRef):
        db = Database(g.db, 'listenitlater')
        obj_deref = Database.dereference(db, obj)
        out = encode_model(obj_deref)
    else:
        raise TypeError, "Could not JSON-encode type '%s': %s" % (type(obj), str(obj))
    return out
Beispiel #41
0
async def get_member(
    database: Database, user_id: str, *, force_refresh: bool = False
) -> typing.Optional[models.DiscordMember]:
    """
    Get a member from the cache, or from the discord API.

    If `force_refresh` is True, the cache is skipped and the entry is updated.
    None may be returned if the member object does not exist.
    """
    collection = database.get_collection("discord_members")

    if force_refresh:
        await collection.delete_one({"user": user_id})

    # `create_index` creates the index if it does not exist, or passes
    # This handles TTL on member objects
    await collection.create_index(
        "inserted_at",
        expireAfterSeconds=60 * 60,  # 1 hour
        name="inserted_at",
    )

    result = await collection.find_one({"user": user_id})

    if result is not None:
        return models.DiscordMember(**json.loads(result["data"]))

    member = await _fetch_member_api(user_id)

    if not member:
        return None

    await collection.insert_one({
        "user": user_id,
        "data": member.json(),
        "inserted_at": datetime.datetime.now(tz=datetime.timezone.utc),
    })
    return member
def test_mongo_session():
    """
    Test if mongo is storing data
    :return:
    """
    from pymongo import MongoClient
    from pymongo.database import Database

    # Parameters for MongoDB connection
    URL = "mongodb"
    PORT = 27017
    DB_NAME = "rasa"

    client = MongoClient(host=URL, port=PORT)
    db = Database(client, DB_NAME)

    if db["sessions"].count_documents({}) > 0:
        print_passed("Session is stored in Mongo DB")
    else:
        print_failed(
            "Nothing found in Mongo, either mongo is not up or chat room input failed"
        )
        exit(-1)
    def __init__(
        self,
        domain,
        host=os.getenv("MONGO_URL") or "mongodb://localhost:27017",
        db="rasa",
        username=None,
        password=None,
        auth_source="admin",
        collection="conversations",
        neo4j_host=os.getenv("NEO4J_URL") or "localhost",
        neo4j_http_port=int(os.getenv("NEO4J_PORT") or 7687),
        neo4j_user=os.getenv("NEO4J_USER") or None,
        neo4j_password=os.getenv("NEO4J_PASSWORD") or None,
        event_broker=None,
    ):
        from pymongo.database import Database
        from pymongo import MongoClient

        self.client = MongoClient(
            host,
            username=username,
            password=password,
            authSource=auth_source,
            connect=False,
        )

        try:
            self.Tracker4J = Tracker4J.Tracker4J(host=neo4j_host,
                                                 port=neo4j_http_port,
                                                 user=neo4j_user,
                                                 password=neo4j_password)
        except:
            self.Tracker4J = None
        self.db = Database(self.client, db)
        self.collection = collection
        super().__init__(domain, event_broker)
        self._ensure_indices()
Beispiel #44
0
    def test(cls, db: database.Database):
        for name in db.list_collection_names():
            print(name)


# def getComputedStressList(examInfo, user, shift=0, pospond=0, db=getDB()):
#     # user = examSession.getUser()
#     gmt_begin = examInfo.startDate - dt.timedelta(minutes=shift)
#     gmt_end = examInfo.endDate + dt.timedelta(minutes=pospond)
#     # convert GMT datetime to GMT timestamp
#     timestamp_begin = unix_time_secs(gmt_begin)
#     timestamp_end = unix_time_secs(gmt_end)
#
#     cs_results = list(db[user.getMatrNr() +'_hrvdata'].find({
#             '_id.timestamp':{
#                 '$gt':timestamp_begin,
#                 '$lt':timestamp_end
#             }
#     }))
#     return cs_results
#
# def getHeartRateList(examInfo, user, shift=0, pospond=0, db=getDB()):
#     # user = examSession.getUser()
#     gmt_begin = examInfo.startDate - dt.timedelta(minutes=shift)
#     gmt_end = examInfo.endDate + dt.timedelta(minutes=pospond)
#     # convert GMT datetime to GMT timestamp
#     timestamp_begin = unix_time_secs(gmt_begin)
#     timestamp_end = unix_time_secs(gmt_end)
#
#     hr_results = list(db[user.getMatrNr() + '_heartrate'].find({
#             '_id.timestamp':{
#                 '$gt':timestamp_begin,
#                 '$lt':timestamp_end
#             }
#     }))
#     return hr_results
Beispiel #45
0
    def test_collection_names(self):
        db = Database(self.client, "pymongo_test")
        db.test.insert_one({"dummy": u"object"})
        db.test.mike.insert_one({"dummy": u"object"})

        colls = db.collection_names()
        self.assertTrue("test" in colls)
        self.assertTrue("test.mike" in colls)
        for coll in colls:
            self.assertTrue("$" not in coll)

        colls_without_systems = db.collection_names(False)
        for coll in colls_without_systems:
            self.assertTrue(not coll.startswith("system."))

        # Force more than one batch.
        db = self.client.many_collections
        for i in range(101):
            db["coll" + str(i)].insert_one({})
        # No Error
        try:
            db.collection_names()
        finally:
            self.client.drop_database("many_collections")
    def test_drop_collection(self):
        db = Database(self.client, "pymongo_test")

        self.assertRaises(TypeError, db.drop_collection, 5)
        self.assertRaises(TypeError, db.drop_collection, None)

        db.test.insert_one({"dummy": u("object")})
        self.assertTrue("test" in db.collection_names())
        db.drop_collection("test")
        self.assertFalse("test" in db.collection_names())

        db.test.insert_one({"dummy": u("object")})
        self.assertTrue("test" in db.collection_names())
        db.drop_collection(u("test"))
        self.assertFalse("test" in db.collection_names())

        db.test.insert_one({"dummy": u("object")})
        self.assertTrue("test" in db.collection_names())
        db.drop_collection(db.test)
        self.assertFalse("test" in db.collection_names())

        db.test.insert_one({"dummy": u("object")})
        self.assertTrue("test" in db.collection_names())
        db.test.drop()
        self.assertFalse("test" in db.collection_names())
        db.test.drop()

        db.drop_collection(db.test.doesnotexist)
    def test_drop_collection(self):
        db = Database(self.connection, "pymongo_test")

        self.assertRaises(TypeError, db.drop_collection, 5)
        self.assertRaises(TypeError, db.drop_collection, None)

        db.test.save({"dummy": u"object"})
        self.assertTrue("test" in db.collection_names())
        db.drop_collection("test")
        self.assertFalse("test" in db.collection_names())

        db.test.save({"dummy": u"object"})
        self.assertTrue("test" in db.collection_names())
        db.drop_collection(u"test")
        self.assertFalse("test" in db.collection_names())

        db.test.save({"dummy": u"object"})
        self.assertTrue("test" in db.collection_names())
        db.drop_collection(db.test)
        self.assertFalse("test" in db.collection_names())

        db.test.save({"dummy": u"object"})
        self.assertTrue("test" in db.collection_names())
        db.test.drop()
        self.assertFalse("test" in db.collection_names())
        db.test.drop()

        db.drop_collection(db.test.doesnotexist)
Beispiel #48
0
 def test_repr(self):
     self.assertEqual(repr(Database(self.connection, "pymongo_test")),
                      "Database(%r, u'pymongo_test')" % self.connection)
    def test_drop_collection(self):
        db = Database(self.connection, "pymongo_test")

        self.assertRaises(TypeError, db.drop_collection, 5)
        self.assertRaises(TypeError, db.drop_collection, None)

        db.test.save({"dummy": u"object"})
        self.assert_("test" in db.collection_names())
        db.drop_collection("test")
        self.assertFalse("test" in db.collection_names())

        db.test.save({"dummy": u"object"})
        self.assert_("test" in db.collection_names())
        db.drop_collection(u"test")
        self.assertFalse("test" in db.collection_names())

        db.test.save({"dummy": u"object"})
        self.assert_("test" in db.collection_names())
        db.drop_collection(db.test)
        self.assertFalse("test" in db.collection_names())

        db.test.save({"dummy": u"object"})
        self.assert_("test" in db.collection_names())
        db.test.drop()
        self.assertFalse("test" in db.collection_names())
        db.test.drop()

        db.drop_collection(db.test.doesnotexist)
 def test_repr(self):
     self.assertEqual(
         repr(Database(self.connection, "pymongo_test")),
         "Database(%r, %s)" % (self.connection, repr(u"pymongo_test")))
 def setUp(self):
     self.db = Database(get_connection(), "pymongo_test")
 def test_get_coll(self):
     db = Database(self.connection, "pymongo_test")
     self.assertEqual(db.test, db["test"])
     self.assertEqual(db.test, Collection(db, "test"))
     self.assertNotEqual(db.test, Collection(db, "mike"))
     self.assertEqual(db.test.mike, db["test.mike"])
Beispiel #53
0
def send(body):
    with pika.BlockingConnection(pika.URLParameters(URL)) as connection:
        channel = connection.channel()
        channel.queue_declare(queue=QUEUE, durable=True)
        channel.basic_publish(exchange='', routing_key=QUEUE, body=body)
        # print(f'[x] Sent {body}')


if __name__ == '__main__':
    """"""
    mongo_url = 'mongodb://192.168.12.102:27017'
    mongo_db = 'company'
    mongo_col = 'items'
    client = MongoClient(mongo_url)
    database = Database(client, mongo_db)
    collection = Collection(database, mongo_col)
    cursor = collection.find(limit=10000000)
    import pytz

    flag = 0
    # for i in cursor:
    #     flag += 1
    #     if flag % 10000 == 0:
    #         print(f'flag: {flag}')
    #     send(json.dumps(i, cls=CustomsJsonEncoder))
    #
    # for i in range(0, 5):
    #     body = {
    #         "a": i,
    #         "b": "2",
class TestCursor(unittest.TestCase):
    def setUp(self):
        self.db = Database(get_connection(), "pymongo_test")

    def test_explain(self):
        a = self.db.test.find()
        b = a.explain()
        for _ in a:
            break
        c = a.explain()
        del b["millis"]
        b.pop("oldPlan", None)
        del c["millis"]
        c.pop("oldPlan", None)
        self.assertEqual(b, c)
        self.assert_("cursor" in b)

    def test_hint(self):
        db = self.db
        self.assertRaises(TypeError, db.test.find().hint, 5.5)
        db.test.remove({})
        db.test.drop_indexes()

        for i in range(100):
            db.test.insert({"num": i, "foo": i})

        self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}).hint([("num", ASCENDING)]).explain)
        self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}).hint([("foo", ASCENDING)]).explain)

        index = db.test.create_index("num")

        spec = [("num", ASCENDING)]
        self.assertEqual(db.test.find({}).explain()["cursor"], "BasicCursor")
        self.assertEqual(db.test.find({}).hint(spec).explain()["cursor"], "BtreeCursor %s" % index)
        self.assertEqual(db.test.find({}).hint(spec).hint(None).explain()["cursor"], "BasicCursor")
        self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}).hint([("foo", ASCENDING)]).explain)

        a = db.test.find({"num": 17})
        a.hint(spec)
        for _ in a:
            break
        self.assertRaises(InvalidOperation, a.hint, spec)

        self.assertRaises(TypeError, db.test.find().hint, index)

    # This is deprecated - test that a warning is actually raised
    def test_slave_okay(self):
        db = self.db
        db.drop_collection("test")
        db.test.save({"x": 1})

        warnings.simplefilter("error")

        self.assertEqual(1, db.test.find().next()["x"])
        self.assertRaises(DeprecationWarning, db.test.find, slave_okay=True)
        self.assertRaises(DeprecationWarning, db.test.find, slave_okay=False)

        warnings.simplefilter("default")

    def test_limit(self):
        db = self.db

        self.assertRaises(TypeError, db.test.find().limit, None)
        self.assertRaises(TypeError, db.test.find().limit, "hello")
        self.assertRaises(TypeError, db.test.find().limit, 5.5)

        db.test.remove({})
        for i in range(100):
            db.test.save({"x": i})

        count = 0
        for _ in db.test.find():
            count += 1
        self.assertEqual(count, 100)

        count = 0
        for _ in db.test.find().limit(20):
            count += 1
        self.assertEqual(count, 20)

        count = 0
        for _ in db.test.find().limit(99):
            count += 1
        self.assertEqual(count, 99)

        count = 0
        for _ in db.test.find().limit(1):
            count += 1
        self.assertEqual(count, 1)

        count = 0
        for _ in db.test.find().limit(0):
            count += 1
        self.assertEqual(count, 100)

        count = 0
        for _ in db.test.find().limit(0).limit(50).limit(10):
            count += 1
        self.assertEqual(count, 10)

        a = db.test.find()
        a.limit(10)
        for _ in a:
            break
        self.assertRaises(InvalidOperation, a.limit, 5)

    def test_skip(self):
        db = self.db

        self.assertRaises(TypeError, db.test.find().skip, None)
        self.assertRaises(TypeError, db.test.find().skip, "hello")
        self.assertRaises(TypeError, db.test.find().skip, 5.5)

        db.drop_collection("test")

        for i in range(100):
            db.test.save({"x": i})

        for i in db.test.find():
            self.assertEqual(i["x"], 0)
            break

        for i in db.test.find().skip(20):
            self.assertEqual(i["x"], 20)
            break

        for i in db.test.find().skip(99):
            self.assertEqual(i["x"], 99)
            break

        for i in db.test.find().skip(1):
            self.assertEqual(i["x"], 1)
            break

        for i in db.test.find().skip(0):
            self.assertEqual(i["x"], 0)
            break

        for i in db.test.find().skip(0).skip(50).skip(10):
            self.assertEqual(i["x"], 10)
            break

        for i in db.test.find().skip(1000):
            self.fail()

        a = db.test.find()
        a.skip(10)
        for _ in a:
            break
        self.assertRaises(InvalidOperation, a.skip, 5)

    def test_sort(self):
        db = self.db

        self.assertRaises(TypeError, db.test.find().sort, 5)
        self.assertRaises(ValueError, db.test.find().sort, [])
        self.assertRaises(TypeError, db.test.find().sort, [], ASCENDING)
        self.assertRaises(TypeError, db.test.find().sort, [("hello", DESCENDING)], DESCENDING)
        self.assertRaises(TypeError, db.test.find().sort, "hello", "world")

        db.test.remove({})

        unsort = range(10)
        random.shuffle(unsort)

        for i in unsort:
            db.test.save({"x": i})

        asc = [i["x"] for i in db.test.find().sort("x", ASCENDING)]
        self.assertEqual(asc, range(10))
        asc = [i["x"] for i in db.test.find().sort("x")]
        self.assertEqual(asc, range(10))
        asc = [i["x"] for i in db.test.find().sort([("x", ASCENDING)])]
        self.assertEqual(asc, range(10))

        expect = range(10)
        expect.reverse()
        desc = [i["x"] for i in db.test.find().sort("x", DESCENDING)]
        self.assertEqual(desc, expect)
        desc = [i["x"] for i in db.test.find().sort([("x", DESCENDING)])]
        self.assertEqual(desc, expect)
        desc = [i["x"] for i in db.test.find().sort("x", ASCENDING).sort("x", DESCENDING)]
        self.assertEqual(desc, expect)

        expected = [(1, 5), (2, 5), (0, 3), (7, 3), (9, 2), (2, 1), (3, 1)]
        shuffled = list(expected)
        random.shuffle(shuffled)

        db.test.remove({})
        for (a, b) in shuffled:
            db.test.save({"a": a, "b": b})

        result = [(i["a"], i["b"]) for i in db.test.find().sort([("b", DESCENDING), ("a", ASCENDING)])]
        self.assertEqual(result, expected)

        a = db.test.find()
        a.sort("x", ASCENDING)
        for _ in a:
            break
        self.assertRaises(InvalidOperation, a.sort, "x", ASCENDING)

    def test_count(self):
        db = self.db
        db.test.remove({})

        self.assertEqual(0, db.test.find().count())

        for i in range(10):
            db.test.save({"x": i})

        self.assertEqual(10, db.test.find().count())
        self.assert_(isinstance(db.test.find().count(), types.IntType))
        self.assertEqual(10, db.test.find().limit(5).count())
        self.assertEqual(10, db.test.find().skip(5).count())

        self.assertEqual(1, db.test.find({"x": 1}).count())
        self.assertEqual(5, db.test.find({"x": {"$lt": 5}}).count())

        a = db.test.find()
        b = a.count()
        for _ in a:
            break
        self.assertEqual(b, a.count())

        self.assertEqual(0, db.test.acollectionthatdoesntexist.find().count())

    def test_where(self):
        db = self.db
        db.test.remove({})

        a = db.test.find()
        self.assertRaises(TypeError, a.where, 5)
        self.assertRaises(TypeError, a.where, None)
        self.assertRaises(TypeError, a.where, {})

        for i in range(10):
            db.test.save({"x": i})

        self.assertEqual(3, len(list(db.test.find().where("this.x < 3"))))
        self.assertEqual(3, len(list(db.test.find().where(Code("this.x < 3")))))
        self.assertEqual(3, len(list(db.test.find().where(Code("this.x < i", {"i": 3})))))
        self.assertEqual(10, len(list(db.test.find())))

        self.assertEqual(3, db.test.find().where("this.x < 3").count())
        self.assertEqual(10, db.test.find().count())
        self.assertEqual(3, db.test.find().where(u"this.x < 3").count())
        self.assertEqual([0, 1, 2], [a["x"] for a in db.test.find().where("this.x < 3")])
        self.assertEqual([], [a["x"] for a in db.test.find({"x": 5}).where("this.x < 3")])
        self.assertEqual([5], [a["x"] for a in db.test.find({"x": 5}).where("this.x > 3")])

        cursor = db.test.find().where("this.x < 3").where("this.x > 7")
        self.assertEqual([8, 9], [a["x"] for a in cursor])

        a = db.test.find()
        b = a.where("this.x > 3")
        for _ in a:
            break
        self.assertRaises(InvalidOperation, a.where, "this.x < 3")

    def test_kill_cursors(self):
        db = self.db
        db.drop_collection("test")

        client_cursors = db._command({"cursorInfo": 1})["clientCursors_size"]
        by_location = db._command({"cursorInfo": 1})["byLocation_size"]

        for i in range(10000):
            db.test.insert({"i": i})

        self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
        self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])

        for _ in range(10):
            db.test.find_one()

        self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
        self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])

        for _ in range(10):
            for x in db.test.find():
                break

        self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
        self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])

        a = db.test.find()
        for x in a:
            break

        self.assertNotEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
        self.assertNotEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])

        del a

        self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
        self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])

        a = db.test.find().limit(10)
        for x in a:
            break

        self.assertEqual(client_cursors, db._command({"cursorInfo": 1})["clientCursors_size"])
        self.assertEqual(by_location, db._command({"cursorInfo": 1})["byLocation_size"])

    def test_rewind(self):
        self.db.test.save({"x": 1})
        self.db.test.save({"x": 2})
        self.db.test.save({"x": 3})

        cursor = self.db.test.find().limit(2)

        count = 0
        for _ in cursor:
            count += 1
        self.assertEqual(2, count)

        count = 0
        for _ in cursor:
            count += 1
        self.assertEqual(0, count)

        cursor.rewind()
        count = 0
        for _ in cursor:
            count += 1
        self.assertEqual(2, count)

        cursor.rewind()
        count = 0
        for _ in cursor:
            break
        cursor.rewind()
        for _ in cursor:
            count += 1
        self.assertEqual(2, count)

        self.assertEqual(cursor, cursor.rewind())

    def test_clone(self):
        self.db.test.save({"x": 1})
        self.db.test.save({"x": 2})
        self.db.test.save({"x": 3})

        cursor = self.db.test.find().limit(2)

        count = 0
        for _ in cursor:
            count += 1
        self.assertEqual(2, count)

        count = 0
        for _ in cursor:
            count += 1
        self.assertEqual(0, count)

        cursor = cursor.clone()
        cursor2 = cursor.clone()
        count = 0
        for _ in cursor:
            count += 1
        self.assertEqual(2, count)
        for _ in cursor2:
            count += 1
        self.assertEqual(4, count)

        cursor.rewind()
        count = 0
        for _ in cursor:
            break
        cursor = cursor.clone()
        for _ in cursor:
            count += 1
        self.assertEqual(2, count)

        self.assertNotEqual(cursor, cursor.clone())

    def test_count_with_fields(self):
        self.db.test.remove({})
        self.db.test.save({"x": 1})

        if not version.at_least(self.db.connection(), (1, 1, 3, -1)):
            for _ in self.db.test.find({}, ["a"]):
                self.fail()

            self.assertEqual(0, self.db.test.find({}, ["a"]).count())
        else:
            self.assertEqual(1, self.db.test.find({}, ["a"]).count())

    def test_bad_getitem(self):
        self.assertRaises(TypeError, lambda x: self.db.test.find()[x], "hello")
        self.assertRaises(TypeError, lambda x: self.db.test.find()[x], 5.5)
        self.assertRaises(TypeError, lambda x: self.db.test.find()[x], None)

    def test_getitem_slice_index(self):
        self.db.drop_collection("test")
        for i in range(100):
            self.db.test.save({"i": i})

        izip = itertools.izip
        count = itertools.count

        self.assertRaises(IndexError, lambda: self.db.test.find()[-1:])
        self.assertRaises(IndexError, lambda: self.db.test.find()[1:2:2])

        for a, b in izip(count(0), self.db.test.find()):
            self.assertEqual(a, b["i"])

        self.assertEqual(100, len(list(self.db.test.find()[0:])))
        for a, b in izip(count(0), self.db.test.find()[0:]):
            self.assertEqual(a, b["i"])

        self.assertEqual(80, len(list(self.db.test.find()[20:])))
        for a, b in izip(count(20), self.db.test.find()[20:]):
            self.assertEqual(a, b["i"])

        for a, b in izip(count(99), self.db.test.find()[99:]):
            self.assertEqual(a, b["i"])

        for i in self.db.test.find()[1000:]:
            self.fail()

        self.assertEqual(5, len(list(self.db.test.find()[20:25])))
        self.assertEqual(5, len(list(self.db.test.find()[20L:25L])))
        for a, b in izip(count(20), self.db.test.find()[20:25]):
            self.assertEqual(a, b["i"])

        self.assertEqual(80, len(list(self.db.test.find()[40:45][20:])))
        for a, b in izip(count(20), self.db.test.find()[40:45][20:]):
            self.assertEqual(a, b["i"])

        self.assertEqual(80, len(list(self.db.test.find()[40:45].limit(0).skip(20))))
        for a, b in izip(count(20), self.db.test.find()[40:45].limit(0).skip(20)):
            self.assertEqual(a, b["i"])

        self.assertEqual(80, len(list(self.db.test.find().limit(10).skip(40)[20:])))
        for a, b in izip(count(20), self.db.test.find().limit(10).skip(40)[20:]):
            self.assertEqual(a, b["i"])

        self.assertEqual(1, len(list(self.db.test.find()[:1])))
        self.assertEqual(5, len(list(self.db.test.find()[:5])))

        self.assertEqual(1, len(list(self.db.test.find()[99:100])))
        self.assertEqual(1, len(list(self.db.test.find()[99:1000])))

        self.assertRaises(IndexError, lambda: self.db.test.find()[10:8])
        self.assertRaises(IndexError, lambda: self.db.test.find()[10:10])
        self.assertRaises(IndexError, lambda: self.db.test.find()[:0])

    def test_getitem_numeric_index(self):
        self.db.drop_collection("test")
        for i in range(100):
            self.db.test.save({"i": i})

        self.assertEqual(0, self.db.test.find()[0]["i"])
        self.assertEqual(50, self.db.test.find()[50]["i"])
        self.assertEqual(50, self.db.test.find().skip(50)[0]["i"])
        self.assertEqual(50, self.db.test.find().skip(49)[1]["i"])
        self.assertEqual(50, self.db.test.find()[50L]["i"])
        self.assertEqual(99, self.db.test.find()[99]["i"])

        self.assertRaises(IndexError, lambda x: self.db.test.find()[x], -1)
        self.assertRaises(IndexError, lambda x: self.db.test.find()[x], 100)
        self.assertRaises(IndexError, lambda x: self.db.test.find().skip(50)[x], 50)

    def test_count_with_limit_and_skip(self):
        if not version.at_least(self.db.connection(), (1, 1, 4, -1)):
            raise SkipTest()

        def check_len(cursor, length):
            self.assertEqual(len(list(cursor)), cursor.count(True))
            self.assertEqual(length, cursor.count(True))

        self.db.drop_collection("test")
        for i in range(100):
            self.db.test.save({"i": i})

        check_len(self.db.test.find(), 100)

        check_len(self.db.test.find().limit(10), 10)
        check_len(self.db.test.find().limit(110), 100)

        check_len(self.db.test.find().skip(10), 90)
        check_len(self.db.test.find().skip(110), 0)

        check_len(self.db.test.find().limit(10).skip(10), 10)
        check_len(self.db.test.find()[10:20], 10)
        check_len(self.db.test.find().limit(10).skip(95), 5)
        check_len(self.db.test.find()[95:105], 5)

    def test_len(self):
        self.assertRaises(TypeError, len, self.db.test.find())

    def test_properties(self):
        self.assertEqual(self.db.test, self.db.test.find().collection)

        def set_coll():
            self.db.test.find().collection = "hello"

        self.assertRaises(AttributeError, set_coll)

    def test_tailable(self):
        db = self.db
        db.drop_collection("test")

        cursor = db.test.find(tailable=True)

        db.test.insert({"x": 1})
        count = 0
        for doc in cursor:
            count += 1
            self.assertEqual(1, doc["x"])
        self.assertEqual(1, count)

        db.test.insert({"x": 2})
        count = 0
        for doc in cursor:
            count += 1
            self.assertEqual(2, doc["x"])
        self.assertEqual(1, count)

        db.test.insert({"x": 3})
        count = 0
        for doc in cursor:
            count += 1
            self.assertEqual(3, doc["x"])
        self.assertEqual(1, count)

        self.assertEqual(3, db.test.count())

    def test_distinct(self):
        if not version.at_least(self.db.connection(), (1, 1, 3, 1)):
            raise SkipTest()

        self.db.drop_collection("test")

        self.db.test.save({"a": 1})
        self.db.test.save({"a": 2})
        self.db.test.save({"a": 2})
        self.db.test.save({"a": 2})
        self.db.test.save({"a": 3})

        distinct = self.db.test.find({"a": {"$lt": 3}}).distinct("a")
        distinct.sort()

        self.assertEqual([1, 2], distinct)

        self.db.drop_collection("test")

        self.db.test.save({"a": {"b": "a"}, "c": 12})
        self.db.test.save({"a": {"b": "b"}, "c": 8})
        self.db.test.save({"a": {"b": "c"}, "c": 12})
        self.db.test.save({"a": {"b": "c"}, "c": 8})

        distinct = self.db.test.find({"c": 8}).distinct("a.b")
        distinct.sort()

        self.assertEqual(["b", "c"], distinct)
    def test_create_collection(self):
        db = Database(self.connection, "pymongo_test")

        db.test.insert({"hello": "world"})
        self.assertRaises(CollectionInvalid, db.create_collection, "test")

        db.drop_collection("test")

        self.assertRaises(TypeError, db.create_collection, 5)
        self.assertRaises(TypeError, db.create_collection, None)
        self.assertRaises(InvalidName, db.create_collection, "coll..ection")

        test = db.create_collection("test")
        test.save({"hello": u"world"})
        self.assertEqual(db.test.find_one()["hello"], "world")
        self.assertTrue(u"test" in db.collection_names())

        db.drop_collection("test.foo")
        db.create_collection("test.foo")
        self.assertTrue(u"test.foo" in db.collection_names())
        self.assertEqual(db.test.foo.options(), {})
        self.assertRaises(CollectionInvalid, db.create_collection, "test.foo")
Beispiel #56
0
 def __init__(self, host, db_name='test', collection_name='fs'):
     self.client = MongoClient(host)
     self.db = Database(self.client, db_name)
     self.fs = GridFS(self.db, collection_name)
            if result.matched_count == 0:
                self.log.warning("Experiement does not exist. Created: " +
                                 experiment_id)
        else:
            self.log.warning("FAILED to update state for experiment: " +
                             experiment_id)


if __name__ == "__main__":
    import yaml
    import logging.config

    with open("logging.yaml", 'r') as logconfig:
        cfg = yaml.safe_load(logconfig)

    logging.config.dictConfig(cfg)
    logger = logging.getLogger('mainLogger')

    url = "10.42.117.13"
    port = 27017
    dbName = "beaglemlDB"
    docName = "duymmyMetrics"

    client = MongoClient(url, port)
    db = Database(client, dbName)

    mw = mongoWriter(db, logger, docName)

    metric = {"time": 1}
    mw.write_metrics(metric)
def getMongoCol():
    uri = "mongodb://%s" % ('localhost:27017')
    client = MongoClient(uri)
    db = Database(client, 'test')
    col = db.get_collection('images')
    return col
# -*- encoding: utf-8 -*-
'''
Created on 2014-11-7

@author: [email protected]
'''

import pymongo
from pymongo.database import Database
from pymongo.mongo_client import MongoClient
from pymongo.collection import Collection

from gridfs import GridFS

if __name__ == '__main__':
    con = pymongo.Connection('172.16.10.170', 27017)
#     con = MongoClient(host='172.16.10.170', port=27017, max_pool_size=200)
            
    db = Database(con, 'blog')
    db.authenticate('admin', 'admin')
    coll = Collection(db, 'foo')
    coll.insert({'a':1, 'b':2, 'c':3})
#     coll.update({'a':1}, {'$set':{'b':4}}, multi=True)
    print [x for x in coll.find()]
#     MongoClient()
#     print user.find({'id':1}).count()
    pass