Exemplo n.º 1
0
 def remove(cls, user=False, contrib=False, draft=False):
     if hasattr(contrib, '_id'):
         _get_db().draft.remove({'a':unicode(user._id),
                                 'c':unicode(contrib._id)})
     elif hasattr(draft, '_id'):
         _get_db().draft.remove({'a':unicode(user._id),
                                 '_id':pymongo.objectid.ObjectId(draft._id)})
Exemplo n.º 2
0
 def touch(cls, contrib):
     """updates all envelopes with new modification date
     """
     _get_db().envelope.update(
         {"contribution": unicode(contrib[contrib._meta["id_field"]])},
         {"$set": {"updated_date": datetime.now(), "title": contrib.title}},
     )
Exemplo n.º 3
0
 def touch(cls, contrib):
     '''updates all envelopes with new modification date
     '''
     _get_db().envelope.update(
         {'contribution': unicode(contrib[contrib._meta['id_field']])},
         {'$set': {
             'updated_date': datetime.now(),
             'title': contrib.title
         }})
Exemplo n.º 4
0
 def remove(cls, user=False, contrib=False, draft=False):
     if hasattr(contrib, '_id'):
         _get_db().draft.remove({
             'a': unicode(user._id),
             'c': unicode(contrib._id)
         })
     elif hasattr(draft, '_id'):
         _get_db().draft.remove({
             'a': unicode(user._id),
             '_id': pymongo.objectid.ObjectId(draft._id)
         })
Exemplo n.º 5
0
    def save_draft(cls, user, contrib):
        """ @param user: required
            @param contrib: ignoring id if blank; always use text
        """
        main_vars = {"a": unicode(user._id)}

        # Only set contribution ID if ID not blank
        if hasattr(contrib, "_id") and contrib._id != "":
            main_vars["c"] = unicode(contrib._id)

        _get_db().draft.update(main_vars, {"$set": {"txt": contrib.get_text(), "t": contrib.title}}, upsert=True)
Exemplo n.º 6
0
 def save_draft(cls, user, contrib):
     ''' @param user: required
         @param contrib: ignoring id if blank; always use text
     '''
     main_vars = {'a':unicode(user._id)}
     
     # Only set contribution ID if ID not blank
     if hasattr(contrib, '_id') and contrib._id != '': 
         main_vars['c'] = unicode(contrib._id)
     
     _get_db().draft.update(main_vars, 
                            {'$set':{'txt':contrib.get_text(),
                                     't':contrib.title}}, 
                            upsert=True)
Exemplo n.º 7
0
 def countcountryenrolments(collectionname, course_id):
     #DO db.clickstream.ensureIndex({country:1})
     #DO db.clickstream.ensureIndex( {event_type: 1,"context.course_id": 1} )
     #DO db.clickstream.ensureIndex({"context.course_id": 1})
     #db.clickstream_tmp.aggregate(    [      { $match : {'context.course_id':'UQx/Think101x/1T2014','event_type':'edx.course.enrollment.activated'} },      { $group : { _id : "$country", thecount: { $sum: 1 } } }    ] )
     results = _get_db()[collectionname].aggregate([{
         "$match": {
             'context.course_id': course_id,
             'event_type': 'edx.course.enrollment.activated'
         }
     }, {
         "$group": {
             "_id": "$country",
             "thecount": {
                 "$sum": 1
             }
         }
     }])
     results = results['result']
     countries = []
     total = 0
     for result in results:
         total += result['thecount']
     for result in results:
         country = {
             'country': result['_id'],
             'count': result['thecount'],
             'percentage': (float(result['thecount']) * 100 / float(total))
         }
         countries.append(country)
     return countries
Exemplo n.º 8
0
 def activeusersdaily(collectionname, course_id):
     results = _get_db()[collectionname].aggregate([{
         "$match": {
             'context.course_id': course_id
         }
     }, {
         "$group": {
             "_id": {
                 "$substr": ["$time", 0, 10]
             },
             "users": {
                 "$addToSet": "$username"
             }
         }
     }, {
         "$unwind": "$users"
     }, {
         "$group": {
             "_id": "$_id",
             "userCount": {
                 "$sum": 1
             }
         }
     }])
     results = results['result']
     for user in results:
         user['date'] = user['_id']
     return results
Exemplo n.º 9
0
 def _flush_db(self):
     from mongoengine.connection import _get_db
     db = _get_db()
     #Truncate/wipe the test database
     names = [name for name in db.collection_names() \
         if 'system.' not in name]
     [db.drop_collection(name) for name in names]
Exemplo n.º 10
0
 def _flush_db(self):
     from mongoengine.connection import _get_db
     me = _get_db()
     #Truncate/wipe the test database
     names = [name for name in me.collection_names() \
         if 'system.' not in name]
     [me.drop_collection(name) for name in names]
Exemplo n.º 11
0
 def setUp(self):
     connect(db='mongoenginetest')
     self.db = _get_db()
     class Person(Document):
         name = StringField()
         age = IntField()
     self.Person = Person
Exemplo n.º 12
0
 def teardown_method(self, method):
     """ teardown any state that was previously setup with a setup_method
     call.
     """
     db = _get_db()
     db.drop_collection('video_info')
     disconnect(alias='test_video')
Exemplo n.º 13
0
def get_db():
    """

    :return:
    :rtype: pymongo.database.Database
    """
    return _get_db()
Exemplo n.º 14
0
def get_app():
    #app = Flask(__name__)
    app = Flask('pyfem')
    app.config.from_object(Config)

    # set environment PYFEM_SETTINGS=testing_settings to automatically use a -test version of db
    if os.environ.get('PYFEM_SETTINGS'):
        if os.environ['PYFEM_SETTINGS'] == 'testing_settings':
            app.config['MONGODB_SETTINGS'] = dict(db=app.config['MONGO_DBNAME'] + '-test')

    app.secret_key = app.config['SECRET_KEY']

    app.me = MongoEngine(app)
    app.pymongo = _get_db()

    # app.jinja_env.add_extension('util.Markdown2Extension')
    # app.jinja_env.filters['slugify'] = slugify
    # app.jinja_env.filters['timesince'] = timesince
    # app.jinja_env.filters['timeuntil'] = timeuntil
    # app.jinja_env.filters['jsonencode'] = jsonencode
    # app.jinja_env.globals['newrelic_head'] = newrelic_head
    # app.jinja_env.globals['newrelic_foot'] = newrelic_foot

    # if not app.config.get('TEMPLATE_DEBUG', True):
    #     compiled_templates = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'compiled_templates')
    #     compiled_files = path.path(compiled_templates).files()
    #     if len(compiled_files) <= 1:
    #         app.jinja_env.compile_templates(compiled_templates, zip=None, py_compile=True)
    #     app.jinja_env.loader = ModuleLoader(compiled_templates)

    configure_logging(app)
    return app
Exemplo n.º 15
0
def test_document_history():

    @save_history
    class Book(Document):
        title = StringField()
        caption = StringField()

    establish_mongo_connection()

    book = Book()
    title_string = "Mother Night"
    book.title = title_string
    book.save()

    history = book.history
    eq_(len(history), 1)
    eq_(history[0]["changes"]["title"], title_string)


    caption_string = "We must be careful about what we pretend to be."

    book.caption = caption_string
    book.save()

    history = book.history

    eq_(len(history), 2)
    eq_(history[1]["changes"]["caption"], caption_string)


    # Clear database
    db = _get_db()
    collection_names = [c for c in db.collection_names() if not c.startswith("system.")]
    for collection in collection_names:
        db.drop_collection(collection)
Exemplo n.º 16
0
def mongodb(collection=None):
    from mongoengine.connection import _get_db
    db = _get_db()
    if collection is not None:
        return getattr(db, collection)
    else:
        return db
Exemplo n.º 17
0
    def factory(cls, id=None, type=None, domain=None, uri=None):
        if id is None and type is None and (domain is None and uri is None):
            return None

        if id or (domain and uri):
            query = {"_id": ObjectId(id)}
            if domain:
                query["D"] = domain
            if domain and uri:
                query["U"] = uri
                del query["_id"]

            values = _get_db().contribution.find_one(query)

            if values is None:
                return None
            values.setdefault("type")
            type = values["type"]
        elif type:
            values = {}

        if type == "html":
            return HtmlContribution._from_son(values)
        elif type == "markdown":
            return MarkdownContribution._from_son(values)
        else:
            return Contribution._from_son(values)
Exemplo n.º 18
0
 def tearDown(self):
     elapsed = time.time() - self._started_at
     print('{} ({}s)'.format(self.id(), round(elapsed, 2)))
     db = _get_db()
     db.client.drop_database(db)
     db.client.close()
     self.app_context.pop()
Exemplo n.º 19
0
def get_jobs_vs_javascript_libraries():
    pipeline = [
        {"$redact": {
            "$cond": {
                "if": {"$eq": ["$deprecated", False]},
                "then": "$$KEEP",
                "else": "$$PRUNE"
            }
        }},
        {"$project": {
            "_id": 0,
            "keywords": 1,
            "deprecated": 1
        }},
        {"$unwind": "$keywords"},
        {"$match": {"keywords.types": {"$in": [keyword_type.types["JS_LIB"]]}}},
        {"$group": {
            "_id": {
                "keyword": "$keywords.keyword",
                "types": "$keywords.types"
            },
            "count": {
                "$sum": 1
            }
        }}
    ]

    db = connection._get_db(reconnect=False)

    return db.job.aggregate(pipeline)
Exemplo n.º 20
0
def get_jobs_vs_programming_languages():
    pipeline = [
        {"$redact": {
            "$cond": {
                "if": {"$eq": ["$deprecated", False]},
                "then": "$$KEEP",
                "else": "$$PRUNE"
            }
        }},
        {"$project": {
            "_id": 0,
            "keywords": 1,
            "deprecated": 1
        }},
        {"$unwind": "$keywords"},
        {"$match": {"keywords.types": {"$in": [keyword_type.types["PROGRAMMING_LANG"]]}}},
        {"$group": {
            "_id": {
                "keyword": "$keywords.keyword",
                "types": "$keywords.types"
            },
            "count": {
                "$sum": 1
            }
        }}
    ]

    db = connection._get_db(reconnect=False)

    return db.job.aggregate(pipeline)
Exemplo n.º 21
0
    def factory(cls, id=None, type=None, domain=None, uri=None):
        if id is None and type is None and (domain is None and uri is None):
            return None

        if id or (domain and uri):
            query = {'_id': pymongo.objectid.ObjectId(id)}
            if domain:
                query['D'] = domain
            if domain and uri:
                query['U'] = uri
                del query['_id']

            values = _get_db().contribution.find_one(query)

            if values is None:
                return None
            values.setdefault('type')
            type = values['type']
        elif type:
            values = {}

        if type == 'html':
            return HtmlContribution._from_son(values)
        elif type == 'markdown':
            return MarkdownContribution._from_son(values)
        else:
            return Contribution._from_son(values)
Exemplo n.º 22
0
 def factory(cls, id=None, type=None, domain=None, uri=None):
     if id is None and type is None and (domain is None and uri is None):
         return None
     
     if id or (domain and uri):
         query = {'_id': pymongo.objectid.ObjectId(id)}
         if domain: 
             query['D'] = domain
         if domain and uri:
             query['U'] = uri
             del query['_id']
         
         values = _get_db().contribution.find_one(query)
         
         if values is None:
             return None
         values.setdefault('type')
         type = values['type']
     elif type:
         values = {}
     
     
     if type == 'html':
         return HtmlContribution._from_son(values)
     elif type == 'markdown':
         return MarkdownContribution._from_son(values)
     else:
         return Contribution._from_son(values)
Exemplo n.º 23
0
    def save_draft(cls, user, contrib):
        ''' @param user: required
            @param contrib: ignoring id if blank; always use text
        '''
        main_vars = {'a': unicode(user._id)}

        # Only set contribution ID if ID not blank
        if hasattr(contrib, '_id') and contrib._id != '':
            main_vars['c'] = unicode(contrib._id)

        _get_db().draft.update(
            main_vars,
            {'$set': {
                'txt': contrib.get_text(),
                't': contrib.title
            }},
            upsert=True)
Exemplo n.º 24
0
def _get_unique_filename(name):
    fs = gridfs.GridFS(_get_db())
    file_root, file_ext = os.path.splitext(name)
    count = itertools.count(1)
    while fs.exists(filename=name):
        # file_ext includes the dot.
        name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext))
    return name
Exemplo n.º 25
0
def _get_unique_filename(name):
    fs = gridfs.GridFS(_get_db())
    file_root, file_ext = os.path.splitext(name)
    count = itertools.count(1)
    while fs.exists(filename=name):
        # file_ext includes the dot.
        name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext))
    return name
Exemplo n.º 26
0
 def _clean(self):
     #from mongoengine.connection import _get_db
     #db = _get_db()
     #db.drop_collection("example")
     clear_document_registry()
     connect(db='mongoenginetest')
     self.db = _get_db()
     self.db.drop_collection("example")
Exemplo n.º 27
0
 def teardown_method(self, method):
     """ teardown any state that was previously setup with a setup_method
     call.
     """
     db = _get_db()
     db.drop_collection('friends')
     db.drop_collection('pending_request')
     disconnect(alias='test')
Exemplo n.º 28
0
 def _clean(self):
     #from mongoengine.connection import _get_db
     #db = _get_db()
     #db.drop_collection("example")
     clear_document_registry()
     connect(db='mongoenginetest')
     self.db = _get_db()
     self.db.drop_collection("example")
Exemplo n.º 29
0
    def setUp(self):
        connect(db='mongoenginetest')
        self.db = _get_db()

        class Person(Document):
            name = StringField()
            age = IntField()

        self.Person = Person
Exemplo n.º 30
0
 def teardown_method(self, method):
     """ teardown any state that was previously setup with a setup_method
     call.
     """
     db = _get_db()
     db.drop_collection('user')
     db.drop_collection('invalid_token')
     db.drop_collection('reset_password_code')
     disconnect(alias='test')
Exemplo n.º 31
0
    def setUp(self):
        connect()
        self.db = _get_db('test')

        @database('test')
        class Person(Document):
            name = StringField()
            age = IntField()
        self.Person = Person
Exemplo n.º 32
0
    def setUp(self):
        connect()
        self.db = _get_db()

        class Person(Document):
            name = StringField()
            age = IntField()
            uid = ObjectIdField()
        self.Person = Person
Exemplo n.º 33
0
 def __init__(self, *args, **kwargs):
     """
     We need to patch jinja_env loader to include flaskext.mongoengine
     templates folder.
     """
     super(MongoenginePanel, self).__init__(*args, **kwargs)
     self.jinja_env.loader = ChoiceLoader([self.jinja_env.loader,
                       PackageLoader('flask_mongoengine', 'templates')])
     self.db = _get_db()
Exemplo n.º 34
0
 def client(self):
     app = create_app(TestConfig)
     db = _get_db()
     app_context = app.app_context()
     app_context.push()
     client = app.test_client()
     yield client
     app_context.pop()
     db.client.drop_database(db)
Exemplo n.º 35
0
 def __init__(self, *args, **kwargs):
     """
     We need to patch jinja_env loader to include flaskext.mongoengine
     templates folder.
     """
     super(MongoenginePanel, self).__init__(*args, **kwargs)
     self.jinja_env.loader = ChoiceLoader([self.jinja_env.loader,
                       PackageLoader('flaskext.mongoengine', 'templates')])
     self.db = _get_db()
Exemplo n.º 36
0
    def __init__(self, level=logging.DEBUG):
        """ Constructor
        
            Keyword Arguments:
            level -- The logging level
        """

        self.log.setLevel(level)
        self.ianatags = self.danatags + " IANA "
        self.fs = GridFS(_get_db())
Exemplo n.º 37
0
    def test_connect(self):
        """Ensure that the connect() method works properly.
        """
        connect('mongoenginetest')

        conn = _get_connection()
        self.assertTrue(isinstance(conn, pymongo.connection.Connection))

        db = _get_db()
        self.assertTrue(isinstance(db, pymongo.database.Database))
        self.assertEqual(db.name, 'mongoenginetest')
Exemplo n.º 38
0
    def setUp(self):
        connect()
        self.db = _get_db()

        class Person(Document):
            name = StringField()
            age = IntField()
            uid = ObjectIdField()
            friends = ListField(StringField())

        self.Person = Person
Exemplo n.º 39
0
    def setUp(self):
        warnings.simplefilter('always', category=UserWarning)
        connect()
        self.db = _get_db()
        mongoengine.base._document_registry = {}

        class Person(Document):
            name = StringField()
            age = IntField(default=20)

        self.person_cls = Person
Exemplo n.º 40
0
    def test_register_connection(self):
        """Ensure that connections with different aliases may be registered.
        """
        register_connection('testdb', 'mongoenginetest2')

        self.assertRaises(ConnectionError, _get_connection)
        conn = _get_connection('testdb')
        self.assertTrue(isinstance(conn, pymongo.connection.Connection))

        db = _get_db('testdb')
        self.assertTrue(isinstance(db, pymongo.database.Database))
        self.assertEqual(db.name, 'mongoenginetest2')
Exemplo n.º 41
0
    def test_encrypted_field(self):
        col = connection._get_db().secret

        # Test creating password
        s = Secret.objects.create(password='******')
        self.assertEqual(s.password, 'hello')
        s.reload()
        self.assertEqual(s.password, 'hello')

        cipher = col.find({'_id': s.id})[0]['password']
        self.assertTrue('hello' not in cipher)
        self.assertTrue(len(cipher) > 16)

        # Test changing password
        s.password = '******'
        s.save()
        s.reload()
        self.assertEqual(s.password, 'other')

        other_cipher = col.find({'_id': s.id})[0]['password']
        self.assertTrue('other' not in other_cipher)
        self.assertTrue(len(other_cipher) > 16)
        self.assertNotEqual(other_cipher, cipher)

        # Make sure password is encrypted differently if we resave.
        s.password = '******'
        s.save()
        s.reload()
        self.assertEqual(s.password, 'hello')

        new_cipher = col.find({'_id': s.id})[0]['password']
        self.assertTrue('hello' not in new_cipher)
        self.assertTrue(len(new_cipher) > 16)
        self.assertNotEqual(new_cipher, cipher)
        self.assertNotEqual(other_cipher, cipher)

        # Test empty password
        s.password = None
        s.save()
        s.reload()
        self.assertEqual(s.password, None)

        raw = col.find({'_id': s.id})[0]
        self.assertTrue('password' not in raw)

        # Test passwords of various lengths
        for pw_len in range(1, 50):
            pw = ''.join(random.choice(string.ascii_letters + string.digits) for x in range(pw_len))
            s = Secret(password=pw)
            s.save()
            s.reload()
            self.assertEqual(s.password, pw)
Exemplo n.º 42
0
 def find_events(collection_name, course_id):
     results = _get_db()[collection_name].aggregate([{
         "$match": {
             'context.course_id': course_id
         }
     }, {
         "$group": {
             "_id": "$username",
             "events": {
                 "$addToSet": "$event_type"
             }
         }
     }])
     return results
Exemplo n.º 43
0
 def setup(self):
     mongoengine.register_connection(mongoengine.DEFAULT_CONNECTION_NAME, TEST_DB_NAME, DB_HOST, DB_PORT)
     # TODO
     # this code is using pymongo
     # maybe mongoengine have the same code
     db = _get_db()
     for name in db.collection_names(False):
         db[name].remove()
     self.cover = tempfile.TemporaryFile()
     self.cover.write('cover')
     self.cover.seek(0)
     self.audio = tempfile.TemporaryFile()
     self.audio.write('audio')
     self.audio.seek(0)
Exemplo n.º 44
0
 def setup(self):
     mongoengine.register_connection(mongoengine.DEFAULT_CONNECTION_NAME,
                                     TEST_DB_NAME, DB_HOST, DB_PORT)
     # TODO
     # this code is using pymongo
     # maybe mongoengine have the same code
     db = _get_db()
     for name in db.collection_names(False):
         db[name].remove()
     self.cover = tempfile.TemporaryFile()
     self.cover.write('cover')
     self.cover.seek(0)
     self.audio = tempfile.TemporaryFile()
     self.audio.write('audio')
     self.audio.seek(0)
def loadFixtures():
    from dummy import DummyFixtures
    from locations import LocationsFixtures
    from mongoengine.connection import _get_db
    from flask import current_app

    # Clear Database before createing data in it
    #db_fd, current_app.config['DATABASE'] = tempfile.mkstemp()
    #current_app.config['TESTING'] = True
    #client = current_app.test_client()
    db = _get_db()
    db.connection.drop_database(current_app.config['MONGODB_SETTINGS']['DB'])

    fixtures = [DummyFixtures(), LocationsFixtures()]
    for fixture in sorted(fixtures, key=lambda f: f.order):
        fixture.load()
Exemplo n.º 46
0
 def countcountryenrolments(collectionname, course_id):
     #DO db.clickstream.ensureIndex({country:1})
     #DO db.clickstream.ensureIndex( {event_type: 1,"context.course_id": 1} )
     #DO db.clickstream.ensureIndex({"context.course_id": 1})
     #db.clickstream_tmp.aggregate(    [      { $match : {'context.course_id':'UQx/Think101x/1T2014','event_type':'edx.course.enrollment.activated'} },      { $group : { _id : "$country", thecount: { $sum: 1 } } }    ] )
     results = _get_db()[collectionname].aggregate([ {"$match":{'context.course_id':course_id,
                                                                'event_type':'edx.course.enrollment.activated'}},{"$group":{"_id": "$country", "thecount": { "$sum": 1 } }} ])
     results = results['result']
     countries = []
     total = 0
     for result in results:
         total += result['thecount']
     for result in results:
         country = {'country': result['_id'], 'count': result['thecount'], 'percentage': (float(result['thecount'])*100/float(total))}
         countries.append(country)
     return countries
Exemplo n.º 47
0
    def countfield(field, withpercentages=False, collectionname="log"):
        total = 0
        reducer = Code("""
             function(cur, result) { result.count += 1 }
        """)
        results = _get_db()[collectionname].group(key={field: 1}, condition={},
                                      initial={"count": 0}, reduce=reducer)
        if withpercentages:
            for result in results:
                total += result['count']

        for result in results:
            result['name'] = result[field]
            if withpercentages:
                result['percentage'] = float(float(result['count'])/float(total))*100
            del result[field]
        return results
Exemplo n.º 48
0
def get_jobs_vs_programs():
    pipeline = [
        {"$redact": {
            "$cond": {
                "if": {"$eq": ["$deprecated", False]},
                "then": "$$KEEP",
                "else": "$$PRUNE"
            }
        }},
        {"$project": {"_id": 0, "programs": 1}},
        {"$unwind": "$programs"},
        {"$group": {"_id": {
            "program": "$programs"
        }, "count": {"$sum": 1}}}
    ]

    db = connection._get_db(reconnect=False)

    return db.job.aggregate(pipeline)
Exemplo n.º 49
0
    def countfield(field, withpercentages=False, collectionname="log"):
        total = 0
        reducer = Code("""
             function(cur, result) { result.count += 1 }
        """)
        results = _get_db()[collectionname].group(key={field: 1},
                                                  condition={},
                                                  initial={"count": 0},
                                                  reduce=reducer)
        if withpercentages:
            for result in results:
                total += result['count']

        for result in results:
            result['name'] = result[field]
            if withpercentages:
                result['percentage'] = float(
                    float(result['count']) / float(total)) * 100
            del result[field]
        return results
Exemplo n.º 50
0
def create_db():
    print("Creating db")
    db_test_name = os.environ["MONGODB_DB"] + "_test"
    db_host = os.environ["MONGODB_HOST"]

    # there's no need to drop the database prior if unhealthy
    # since it's dropped automatically after tests are run

    # TESTING flag disables error catching during request handling,
    # so that you get better error reports when performing test requests
    # against the application.
    yield {
        "MONGODB_HOST": db_host,
        "MONGODB_DB": db_test_name,
        "TESTING": True,
    }

    print("Destroying db")
    db = _get_db()
    db.client.drop_database(db)
Exemplo n.º 51
0
def get_jobs_vs_locations():
    pipeline = [
        {"$redact": {
            "$cond": {
                "if": {"$eq": ["$deprecated", False]},
                "then": "$$KEEP",
                "else": "$$PRUNE"
            }
        }},
        {"$project": {"_id": 0, "location": 1}},
        {"$unwind": "$location"},
        {"$group": {"_id": {
            "location": "$location.name",
            "longitude": "$location.longitude",
            "latitude": "$location.latitude"
        }}}
    ]

    db = connection._get_db(reconnect=False)

    return db.job.aggregate(pipeline)
def importFile(filePath):
    f = open(filePath, "r", encoding="ISO-8859-1")

    db = _get_db()
    if 'ipToCountry_tmp' in db.collection_names(False):
        db.drop_collection('ipToCountry_tmp')

    i = 0
    for line in f:
        if line[0] == '#':
            continue

        r = list(csv.reader([line]))[0]

        doc = IpToCountryTmp(
            ipFrom=int(r[0]),
            ipTo=int(r[1]),
            registry=r[2],
            assigned=time.strftime('%Y-%m-%d %H:%M:%S',
                                   time.localtime(int(r[3]))),
            ctry=r[4],
            cntry=r[5],
            country=r[6],
        )

        doc.save()

        # if i==10:
        #     break
        # i += 1

    f.close()

    if 'ipToCountry_bak' in db.collection_names(False):
        db.drop_collection('ipToCountry_bak')
    if 'ipToCountry' in db.collection_names(False):
        db["ipToCountry"].rename("ipToCountry_bak")
    db["ipToCountry_tmp"].rename("ipToCountry")
Exemplo n.º 53
0
    def default(self, value):
        """Convert niche data types, and ``Document`` subclasses.
        
        Conversion notes:

        - ``datetime.date`` and ``datetime.datetime`` objects are
          converted into UNIX timestamps.
        - ``pymongo.objectid.ObjectId`` objects are converted into
          ``unicode`` strings.
        - ``mongoengine.document.Document`` instances are converted
          into ``dict`` representations via instance._data.
        - ``DBRef`` objects are automagically dereferenced.
        """

        if isinstance(value, datetime):
            return mktime(value.timetuple())
        elif isinstance(value, date):
            dt = datetime(value.year, value.month, value.day, 0, 0, 0)
            return mktime(dt.timetuple())
        elif isinstance(value, ObjectId):
            return unicode(value)
        elif isinstance(value, (Cursor, QuerySet)):
            return list(value)
        elif issubclass(value.__class__, (Document, EmbeddedDocument)):
            present_keys = dict(filter(lambda k: k[0] in value._present_fields, value.__dict__["_data"].iteritems()))
            return present_keys
        elif isinstance(value, DBRef):
            # dereference
            obj = _get_db().dereference(value)
            if obj:
                del obj["_types"]
                return obj
        elif isinstance(value, LazyReference):
            return value.dereference()
        else:
            # use no special encoding and hope for the best
            return value
Exemplo n.º 54
0
 def eventtypes(collectionname, course_id):
     results = _get_db()[collectionname].aggregate([{
         "$match": {
             'context.course_id': course_id
         }
     }, {
         "$group": {
             "_id": {
                 "type": "$event_type",
                 "date": {
                     "$concat": [{
                         "$substr": [{
                             "$year": ['$time_date']
                         }, 0, 4]
                     }, "_", {
                         "$substr": [{
                             "$week": ['$time_date']
                         }, 0, 2]
                     }]
                 }
             },
             "users": {
                 "$addToSet": "$username"
             }
         }
     }, {
         "$unwind": "$users"
     }, {
         "$group": {
             "_id": "$_id",
             "userCount": {
                 "$sum": 1
             }
         }
     }])
     results = results['result']
     return results
Exemplo n.º 55
0
 def tearDown(self):
     db = _get_db()
     db.client.drop_database(db)
Exemplo n.º 56
0
 def tearDown(self):
     db = _get_db()
     collections = db.collection_names(include_system_collections=False)
     for collection in collections:
         db[collection].drop()
Exemplo n.º 57
0
 def tearDownClass(cls):
     db = _get_db()
     db.client.drop_database(db)
Exemplo n.º 58
0
 def setUp(self):
     connect(db='mongoenginetest')
     self.db = _get_db()
 def tearDown(self):
     db = _get_db()
     db.drop_collection("todo")