def remove(cls, user=False, contrib=False, draft=False): if hasattr(contrib, '_id'): _get_db().draft.remove({'a':unicode(user._id), 'c':unicode(contrib._id)}) elif hasattr(draft, '_id'): _get_db().draft.remove({'a':unicode(user._id), '_id':pymongo.objectid.ObjectId(draft._id)})
def touch(cls, contrib): """updates all envelopes with new modification date """ _get_db().envelope.update( {"contribution": unicode(contrib[contrib._meta["id_field"]])}, {"$set": {"updated_date": datetime.now(), "title": contrib.title}}, )
def touch(cls, contrib): '''updates all envelopes with new modification date ''' _get_db().envelope.update( {'contribution': unicode(contrib[contrib._meta['id_field']])}, {'$set': { 'updated_date': datetime.now(), 'title': contrib.title }})
def remove(cls, user=False, contrib=False, draft=False): if hasattr(contrib, '_id'): _get_db().draft.remove({ 'a': unicode(user._id), 'c': unicode(contrib._id) }) elif hasattr(draft, '_id'): _get_db().draft.remove({ 'a': unicode(user._id), '_id': pymongo.objectid.ObjectId(draft._id) })
def save_draft(cls, user, contrib): """ @param user: required @param contrib: ignoring id if blank; always use text """ main_vars = {"a": unicode(user._id)} # Only set contribution ID if ID not blank if hasattr(contrib, "_id") and contrib._id != "": main_vars["c"] = unicode(contrib._id) _get_db().draft.update(main_vars, {"$set": {"txt": contrib.get_text(), "t": contrib.title}}, upsert=True)
def save_draft(cls, user, contrib): ''' @param user: required @param contrib: ignoring id if blank; always use text ''' main_vars = {'a':unicode(user._id)} # Only set contribution ID if ID not blank if hasattr(contrib, '_id') and contrib._id != '': main_vars['c'] = unicode(contrib._id) _get_db().draft.update(main_vars, {'$set':{'txt':contrib.get_text(), 't':contrib.title}}, upsert=True)
def countcountryenrolments(collectionname, course_id): #DO db.clickstream.ensureIndex({country:1}) #DO db.clickstream.ensureIndex( {event_type: 1,"context.course_id": 1} ) #DO db.clickstream.ensureIndex({"context.course_id": 1}) #db.clickstream_tmp.aggregate( [ { $match : {'context.course_id':'UQx/Think101x/1T2014','event_type':'edx.course.enrollment.activated'} }, { $group : { _id : "$country", thecount: { $sum: 1 } } } ] ) results = _get_db()[collectionname].aggregate([{ "$match": { 'context.course_id': course_id, 'event_type': 'edx.course.enrollment.activated' } }, { "$group": { "_id": "$country", "thecount": { "$sum": 1 } } }]) results = results['result'] countries = [] total = 0 for result in results: total += result['thecount'] for result in results: country = { 'country': result['_id'], 'count': result['thecount'], 'percentage': (float(result['thecount']) * 100 / float(total)) } countries.append(country) return countries
def activeusersdaily(collectionname, course_id): results = _get_db()[collectionname].aggregate([{ "$match": { 'context.course_id': course_id } }, { "$group": { "_id": { "$substr": ["$time", 0, 10] }, "users": { "$addToSet": "$username" } } }, { "$unwind": "$users" }, { "$group": { "_id": "$_id", "userCount": { "$sum": 1 } } }]) results = results['result'] for user in results: user['date'] = user['_id'] return results
def _flush_db(self): from mongoengine.connection import _get_db db = _get_db() #Truncate/wipe the test database names = [name for name in db.collection_names() \ if 'system.' not in name] [db.drop_collection(name) for name in names]
def _flush_db(self): from mongoengine.connection import _get_db me = _get_db() #Truncate/wipe the test database names = [name for name in me.collection_names() \ if 'system.' not in name] [me.drop_collection(name) for name in names]
def setUp(self): connect(db='mongoenginetest') self.db = _get_db() class Person(Document): name = StringField() age = IntField() self.Person = Person
def teardown_method(self, method): """ teardown any state that was previously setup with a setup_method call. """ db = _get_db() db.drop_collection('video_info') disconnect(alias='test_video')
def get_db(): """ :return: :rtype: pymongo.database.Database """ return _get_db()
def get_app(): #app = Flask(__name__) app = Flask('pyfem') app.config.from_object(Config) # set environment PYFEM_SETTINGS=testing_settings to automatically use a -test version of db if os.environ.get('PYFEM_SETTINGS'): if os.environ['PYFEM_SETTINGS'] == 'testing_settings': app.config['MONGODB_SETTINGS'] = dict(db=app.config['MONGO_DBNAME'] + '-test') app.secret_key = app.config['SECRET_KEY'] app.me = MongoEngine(app) app.pymongo = _get_db() # app.jinja_env.add_extension('util.Markdown2Extension') # app.jinja_env.filters['slugify'] = slugify # app.jinja_env.filters['timesince'] = timesince # app.jinja_env.filters['timeuntil'] = timeuntil # app.jinja_env.filters['jsonencode'] = jsonencode # app.jinja_env.globals['newrelic_head'] = newrelic_head # app.jinja_env.globals['newrelic_foot'] = newrelic_foot # if not app.config.get('TEMPLATE_DEBUG', True): # compiled_templates = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'compiled_templates') # compiled_files = path.path(compiled_templates).files() # if len(compiled_files) <= 1: # app.jinja_env.compile_templates(compiled_templates, zip=None, py_compile=True) # app.jinja_env.loader = ModuleLoader(compiled_templates) configure_logging(app) return app
def test_document_history(): @save_history class Book(Document): title = StringField() caption = StringField() establish_mongo_connection() book = Book() title_string = "Mother Night" book.title = title_string book.save() history = book.history eq_(len(history), 1) eq_(history[0]["changes"]["title"], title_string) caption_string = "We must be careful about what we pretend to be." book.caption = caption_string book.save() history = book.history eq_(len(history), 2) eq_(history[1]["changes"]["caption"], caption_string) # Clear database db = _get_db() collection_names = [c for c in db.collection_names() if not c.startswith("system.")] for collection in collection_names: db.drop_collection(collection)
def mongodb(collection=None): from mongoengine.connection import _get_db db = _get_db() if collection is not None: return getattr(db, collection) else: return db
def factory(cls, id=None, type=None, domain=None, uri=None): if id is None and type is None and (domain is None and uri is None): return None if id or (domain and uri): query = {"_id": ObjectId(id)} if domain: query["D"] = domain if domain and uri: query["U"] = uri del query["_id"] values = _get_db().contribution.find_one(query) if values is None: return None values.setdefault("type") type = values["type"] elif type: values = {} if type == "html": return HtmlContribution._from_son(values) elif type == "markdown": return MarkdownContribution._from_son(values) else: return Contribution._from_son(values)
def tearDown(self): elapsed = time.time() - self._started_at print('{} ({}s)'.format(self.id(), round(elapsed, 2))) db = _get_db() db.client.drop_database(db) db.client.close() self.app_context.pop()
def get_jobs_vs_javascript_libraries(): pipeline = [ {"$redact": { "$cond": { "if": {"$eq": ["$deprecated", False]}, "then": "$$KEEP", "else": "$$PRUNE" } }}, {"$project": { "_id": 0, "keywords": 1, "deprecated": 1 }}, {"$unwind": "$keywords"}, {"$match": {"keywords.types": {"$in": [keyword_type.types["JS_LIB"]]}}}, {"$group": { "_id": { "keyword": "$keywords.keyword", "types": "$keywords.types" }, "count": { "$sum": 1 } }} ] db = connection._get_db(reconnect=False) return db.job.aggregate(pipeline)
def get_jobs_vs_programming_languages(): pipeline = [ {"$redact": { "$cond": { "if": {"$eq": ["$deprecated", False]}, "then": "$$KEEP", "else": "$$PRUNE" } }}, {"$project": { "_id": 0, "keywords": 1, "deprecated": 1 }}, {"$unwind": "$keywords"}, {"$match": {"keywords.types": {"$in": [keyword_type.types["PROGRAMMING_LANG"]]}}}, {"$group": { "_id": { "keyword": "$keywords.keyword", "types": "$keywords.types" }, "count": { "$sum": 1 } }} ] db = connection._get_db(reconnect=False) return db.job.aggregate(pipeline)
def factory(cls, id=None, type=None, domain=None, uri=None): if id is None and type is None and (domain is None and uri is None): return None if id or (domain and uri): query = {'_id': pymongo.objectid.ObjectId(id)} if domain: query['D'] = domain if domain and uri: query['U'] = uri del query['_id'] values = _get_db().contribution.find_one(query) if values is None: return None values.setdefault('type') type = values['type'] elif type: values = {} if type == 'html': return HtmlContribution._from_son(values) elif type == 'markdown': return MarkdownContribution._from_son(values) else: return Contribution._from_son(values)
def save_draft(cls, user, contrib): ''' @param user: required @param contrib: ignoring id if blank; always use text ''' main_vars = {'a': unicode(user._id)} # Only set contribution ID if ID not blank if hasattr(contrib, '_id') and contrib._id != '': main_vars['c'] = unicode(contrib._id) _get_db().draft.update( main_vars, {'$set': { 'txt': contrib.get_text(), 't': contrib.title }}, upsert=True)
def _get_unique_filename(name): fs = gridfs.GridFS(_get_db()) file_root, file_ext = os.path.splitext(name) count = itertools.count(1) while fs.exists(filename=name): # file_ext includes the dot. name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext)) return name
def _clean(self): #from mongoengine.connection import _get_db #db = _get_db() #db.drop_collection("example") clear_document_registry() connect(db='mongoenginetest') self.db = _get_db() self.db.drop_collection("example")
def teardown_method(self, method): """ teardown any state that was previously setup with a setup_method call. """ db = _get_db() db.drop_collection('friends') db.drop_collection('pending_request') disconnect(alias='test')
def teardown_method(self, method): """ teardown any state that was previously setup with a setup_method call. """ db = _get_db() db.drop_collection('user') db.drop_collection('invalid_token') db.drop_collection('reset_password_code') disconnect(alias='test')
def setUp(self): connect() self.db = _get_db('test') @database('test') class Person(Document): name = StringField() age = IntField() self.Person = Person
def setUp(self): connect() self.db = _get_db() class Person(Document): name = StringField() age = IntField() uid = ObjectIdField() self.Person = Person
def __init__(self, *args, **kwargs): """ We need to patch jinja_env loader to include flaskext.mongoengine templates folder. """ super(MongoenginePanel, self).__init__(*args, **kwargs) self.jinja_env.loader = ChoiceLoader([self.jinja_env.loader, PackageLoader('flask_mongoengine', 'templates')]) self.db = _get_db()
def client(self): app = create_app(TestConfig) db = _get_db() app_context = app.app_context() app_context.push() client = app.test_client() yield client app_context.pop() db.client.drop_database(db)
def __init__(self, *args, **kwargs): """ We need to patch jinja_env loader to include flaskext.mongoengine templates folder. """ super(MongoenginePanel, self).__init__(*args, **kwargs) self.jinja_env.loader = ChoiceLoader([self.jinja_env.loader, PackageLoader('flaskext.mongoengine', 'templates')]) self.db = _get_db()
def __init__(self, level=logging.DEBUG): """ Constructor Keyword Arguments: level -- The logging level """ self.log.setLevel(level) self.ianatags = self.danatags + " IANA " self.fs = GridFS(_get_db())
def test_connect(self): """Ensure that the connect() method works properly. """ connect('mongoenginetest') conn = _get_connection() self.assertTrue(isinstance(conn, pymongo.connection.Connection)) db = _get_db() self.assertTrue(isinstance(db, pymongo.database.Database)) self.assertEqual(db.name, 'mongoenginetest')
def setUp(self): connect() self.db = _get_db() class Person(Document): name = StringField() age = IntField() uid = ObjectIdField() friends = ListField(StringField()) self.Person = Person
def setUp(self): warnings.simplefilter('always', category=UserWarning) connect() self.db = _get_db() mongoengine.base._document_registry = {} class Person(Document): name = StringField() age = IntField(default=20) self.person_cls = Person
def test_register_connection(self): """Ensure that connections with different aliases may be registered. """ register_connection('testdb', 'mongoenginetest2') self.assertRaises(ConnectionError, _get_connection) conn = _get_connection('testdb') self.assertTrue(isinstance(conn, pymongo.connection.Connection)) db = _get_db('testdb') self.assertTrue(isinstance(db, pymongo.database.Database)) self.assertEqual(db.name, 'mongoenginetest2')
def test_encrypted_field(self): col = connection._get_db().secret # Test creating password s = Secret.objects.create(password='******') self.assertEqual(s.password, 'hello') s.reload() self.assertEqual(s.password, 'hello') cipher = col.find({'_id': s.id})[0]['password'] self.assertTrue('hello' not in cipher) self.assertTrue(len(cipher) > 16) # Test changing password s.password = '******' s.save() s.reload() self.assertEqual(s.password, 'other') other_cipher = col.find({'_id': s.id})[0]['password'] self.assertTrue('other' not in other_cipher) self.assertTrue(len(other_cipher) > 16) self.assertNotEqual(other_cipher, cipher) # Make sure password is encrypted differently if we resave. s.password = '******' s.save() s.reload() self.assertEqual(s.password, 'hello') new_cipher = col.find({'_id': s.id})[0]['password'] self.assertTrue('hello' not in new_cipher) self.assertTrue(len(new_cipher) > 16) self.assertNotEqual(new_cipher, cipher) self.assertNotEqual(other_cipher, cipher) # Test empty password s.password = None s.save() s.reload() self.assertEqual(s.password, None) raw = col.find({'_id': s.id})[0] self.assertTrue('password' not in raw) # Test passwords of various lengths for pw_len in range(1, 50): pw = ''.join(random.choice(string.ascii_letters + string.digits) for x in range(pw_len)) s = Secret(password=pw) s.save() s.reload() self.assertEqual(s.password, pw)
def find_events(collection_name, course_id): results = _get_db()[collection_name].aggregate([{ "$match": { 'context.course_id': course_id } }, { "$group": { "_id": "$username", "events": { "$addToSet": "$event_type" } } }]) return results
def setup(self): mongoengine.register_connection(mongoengine.DEFAULT_CONNECTION_NAME, TEST_DB_NAME, DB_HOST, DB_PORT) # TODO # this code is using pymongo # maybe mongoengine have the same code db = _get_db() for name in db.collection_names(False): db[name].remove() self.cover = tempfile.TemporaryFile() self.cover.write('cover') self.cover.seek(0) self.audio = tempfile.TemporaryFile() self.audio.write('audio') self.audio.seek(0)
def loadFixtures(): from dummy import DummyFixtures from locations import LocationsFixtures from mongoengine.connection import _get_db from flask import current_app # Clear Database before createing data in it #db_fd, current_app.config['DATABASE'] = tempfile.mkstemp() #current_app.config['TESTING'] = True #client = current_app.test_client() db = _get_db() db.connection.drop_database(current_app.config['MONGODB_SETTINGS']['DB']) fixtures = [DummyFixtures(), LocationsFixtures()] for fixture in sorted(fixtures, key=lambda f: f.order): fixture.load()
def countcountryenrolments(collectionname, course_id): #DO db.clickstream.ensureIndex({country:1}) #DO db.clickstream.ensureIndex( {event_type: 1,"context.course_id": 1} ) #DO db.clickstream.ensureIndex({"context.course_id": 1}) #db.clickstream_tmp.aggregate( [ { $match : {'context.course_id':'UQx/Think101x/1T2014','event_type':'edx.course.enrollment.activated'} }, { $group : { _id : "$country", thecount: { $sum: 1 } } } ] ) results = _get_db()[collectionname].aggregate([ {"$match":{'context.course_id':course_id, 'event_type':'edx.course.enrollment.activated'}},{"$group":{"_id": "$country", "thecount": { "$sum": 1 } }} ]) results = results['result'] countries = [] total = 0 for result in results: total += result['thecount'] for result in results: country = {'country': result['_id'], 'count': result['thecount'], 'percentage': (float(result['thecount'])*100/float(total))} countries.append(country) return countries
def countfield(field, withpercentages=False, collectionname="log"): total = 0 reducer = Code(""" function(cur, result) { result.count += 1 } """) results = _get_db()[collectionname].group(key={field: 1}, condition={}, initial={"count": 0}, reduce=reducer) if withpercentages: for result in results: total += result['count'] for result in results: result['name'] = result[field] if withpercentages: result['percentage'] = float(float(result['count'])/float(total))*100 del result[field] return results
def get_jobs_vs_programs(): pipeline = [ {"$redact": { "$cond": { "if": {"$eq": ["$deprecated", False]}, "then": "$$KEEP", "else": "$$PRUNE" } }}, {"$project": {"_id": 0, "programs": 1}}, {"$unwind": "$programs"}, {"$group": {"_id": { "program": "$programs" }, "count": {"$sum": 1}}} ] db = connection._get_db(reconnect=False) return db.job.aggregate(pipeline)
def countfield(field, withpercentages=False, collectionname="log"): total = 0 reducer = Code(""" function(cur, result) { result.count += 1 } """) results = _get_db()[collectionname].group(key={field: 1}, condition={}, initial={"count": 0}, reduce=reducer) if withpercentages: for result in results: total += result['count'] for result in results: result['name'] = result[field] if withpercentages: result['percentage'] = float( float(result['count']) / float(total)) * 100 del result[field] return results
def create_db(): print("Creating db") db_test_name = os.environ["MONGODB_DB"] + "_test" db_host = os.environ["MONGODB_HOST"] # there's no need to drop the database prior if unhealthy # since it's dropped automatically after tests are run # TESTING flag disables error catching during request handling, # so that you get better error reports when performing test requests # against the application. yield { "MONGODB_HOST": db_host, "MONGODB_DB": db_test_name, "TESTING": True, } print("Destroying db") db = _get_db() db.client.drop_database(db)
def get_jobs_vs_locations(): pipeline = [ {"$redact": { "$cond": { "if": {"$eq": ["$deprecated", False]}, "then": "$$KEEP", "else": "$$PRUNE" } }}, {"$project": {"_id": 0, "location": 1}}, {"$unwind": "$location"}, {"$group": {"_id": { "location": "$location.name", "longitude": "$location.longitude", "latitude": "$location.latitude" }}} ] db = connection._get_db(reconnect=False) return db.job.aggregate(pipeline)
def importFile(filePath): f = open(filePath, "r", encoding="ISO-8859-1") db = _get_db() if 'ipToCountry_tmp' in db.collection_names(False): db.drop_collection('ipToCountry_tmp') i = 0 for line in f: if line[0] == '#': continue r = list(csv.reader([line]))[0] doc = IpToCountryTmp( ipFrom=int(r[0]), ipTo=int(r[1]), registry=r[2], assigned=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(r[3]))), ctry=r[4], cntry=r[5], country=r[6], ) doc.save() # if i==10: # break # i += 1 f.close() if 'ipToCountry_bak' in db.collection_names(False): db.drop_collection('ipToCountry_bak') if 'ipToCountry' in db.collection_names(False): db["ipToCountry"].rename("ipToCountry_bak") db["ipToCountry_tmp"].rename("ipToCountry")
def default(self, value): """Convert niche data types, and ``Document`` subclasses. Conversion notes: - ``datetime.date`` and ``datetime.datetime`` objects are converted into UNIX timestamps. - ``pymongo.objectid.ObjectId`` objects are converted into ``unicode`` strings. - ``mongoengine.document.Document`` instances are converted into ``dict`` representations via instance._data. - ``DBRef`` objects are automagically dereferenced. """ if isinstance(value, datetime): return mktime(value.timetuple()) elif isinstance(value, date): dt = datetime(value.year, value.month, value.day, 0, 0, 0) return mktime(dt.timetuple()) elif isinstance(value, ObjectId): return unicode(value) elif isinstance(value, (Cursor, QuerySet)): return list(value) elif issubclass(value.__class__, (Document, EmbeddedDocument)): present_keys = dict(filter(lambda k: k[0] in value._present_fields, value.__dict__["_data"].iteritems())) return present_keys elif isinstance(value, DBRef): # dereference obj = _get_db().dereference(value) if obj: del obj["_types"] return obj elif isinstance(value, LazyReference): return value.dereference() else: # use no special encoding and hope for the best return value
def eventtypes(collectionname, course_id): results = _get_db()[collectionname].aggregate([{ "$match": { 'context.course_id': course_id } }, { "$group": { "_id": { "type": "$event_type", "date": { "$concat": [{ "$substr": [{ "$year": ['$time_date'] }, 0, 4] }, "_", { "$substr": [{ "$week": ['$time_date'] }, 0, 2] }] } }, "users": { "$addToSet": "$username" } } }, { "$unwind": "$users" }, { "$group": { "_id": "$_id", "userCount": { "$sum": 1 } } }]) results = results['result'] return results
def tearDown(self): db = _get_db() db.client.drop_database(db)
def tearDown(self): db = _get_db() collections = db.collection_names(include_system_collections=False) for collection in collections: db[collection].drop()
def tearDownClass(cls): db = _get_db() db.client.drop_database(db)
def setUp(self): connect(db='mongoenginetest') self.db = _get_db()
def tearDown(self): db = _get_db() db.drop_collection("todo")