def remove(cls, user=False, contrib=False, draft=False):
if hasattr(contrib, '_id'):
_get_db().draft.remove({'a':unicode(user._id),
'c':unicode(contrib._id)})
elif hasattr(draft, '_id'):
_get_db().draft.remove({'a':unicode(user._id),
'_id':pymongo.objectid.ObjectId(draft._id)})
def touch(cls, contrib):
"""updates all envelopes with new modification date
"""
_get_db().envelope.update(
{"contribution": unicode(contrib[contrib._meta["id_field"]])},
{"$set": {"updated_date": datetime.now(), "title": contrib.title}},
)
def touch(cls, contrib):
'''updates all envelopes with new modification date
'''
_get_db().envelope.update(
{'contribution': unicode(contrib[contrib._meta['id_field']])},
{'$set': {
'updated_date': datetime.now(),
'title': contrib.title
}})
def remove(cls, user=False, contrib=False, draft=False):
if hasattr(contrib, '_id'):
_get_db().draft.remove({
'a': unicode(user._id),
'c': unicode(contrib._id)
})
elif hasattr(draft, '_id'):
_get_db().draft.remove({
'a': unicode(user._id),
'_id': pymongo.objectid.ObjectId(draft._id)
})
def save_draft(cls, user, contrib):
""" @param user: required
@param contrib: ignoring id if blank; always use text
"""
main_vars = {"a": unicode(user._id)}
# Only set contribution ID if ID not blank
if hasattr(contrib, "_id") and contrib._id != "":
main_vars["c"] = unicode(contrib._id)
_get_db().draft.update(main_vars, {"$set": {"txt": contrib.get_text(), "t": contrib.title}}, upsert=True)
def save_draft(cls, user, contrib):
''' @param user: required
@param contrib: ignoring id if blank; always use text
'''
main_vars = {'a':unicode(user._id)}
# Only set contribution ID if ID not blank
if hasattr(contrib, '_id') and contrib._id != '':
main_vars['c'] = unicode(contrib._id)
_get_db().draft.update(main_vars,
{'$set':{'txt':contrib.get_text(),
't':contrib.title}},
upsert=True)
def countcountryenrolments(collectionname, course_id):
#DO db.clickstream.ensureIndex({country:1})
#DO db.clickstream.ensureIndex( {event_type: 1,"context.course_id": 1} )
#DO db.clickstream.ensureIndex({"context.course_id": 1})
#db.clickstream_tmp.aggregate( [ { $match : {'context.course_id':'UQx/Think101x/1T2014','event_type':'edx.course.enrollment.activated'} }, { $group : { _id : "$country", thecount: { $sum: 1 } } } ] )
results = _get_db()[collectionname].aggregate([{
"$match": {
'context.course_id': course_id,
'event_type': 'edx.course.enrollment.activated'
}
}, {
"$group": {
"_id": "$country",
"thecount": {
"$sum": 1
}
}
}])
results = results['result']
countries = []
total = 0
for result in results:
total += result['thecount']
for result in results:
country = {
'country': result['_id'],
'count': result['thecount'],
'percentage': (float(result['thecount']) * 100 / float(total))
}
countries.append(country)
return countries
def activeusersdaily(collectionname, course_id):
results = _get_db()[collectionname].aggregate([{
"$match": {
'context.course_id': course_id
}
}, {
"$group": {
"_id": {
"$substr": ["$time", 0, 10]
},
"users": {
"$addToSet": "$username"
}
}
}, {
"$unwind": "$users"
}, {
"$group": {
"_id": "$_id",
"userCount": {
"$sum": 1
}
}
}])
results = results['result']
for user in results:
user['date'] = user['_id']
return results
def _flush_db(self):
from mongoengine.connection import _get_db
db = _get_db()
#Truncate/wipe the test database
names = [name for name in db.collection_names() \
if 'system.' not in name]
[db.drop_collection(name) for name in names]
def _flush_db(self):
from mongoengine.connection import _get_db
me = _get_db()
#Truncate/wipe the test database
names = [name for name in me.collection_names() \
if 'system.' not in name]
[me.drop_collection(name) for name in names]
def setUp(self):
connect(db='mongoenginetest')
self.db = _get_db()
class Person(Document):
name = StringField()
age = IntField()
self.Person = Person
def teardown_method(self, method):
""" teardown any state that was previously setup with a setup_method
call.
"""
db = _get_db()
db.drop_collection('video_info')
disconnect(alias='test_video')
def get_db():
"""
:return:
:rtype: pymongo.database.Database
"""
return _get_db()
def get_app():
#app = Flask(__name__)
app = Flask('pyfem')
app.config.from_object(Config)
# set environment PYFEM_SETTINGS=testing_settings to automatically use a -test version of db
if os.environ.get('PYFEM_SETTINGS'):
if os.environ['PYFEM_SETTINGS'] == 'testing_settings':
app.config['MONGODB_SETTINGS'] = dict(db=app.config['MONGO_DBNAME'] + '-test')
app.secret_key = app.config['SECRET_KEY']
app.me = MongoEngine(app)
app.pymongo = _get_db()
# app.jinja_env.add_extension('util.Markdown2Extension')
# app.jinja_env.filters['slugify'] = slugify
# app.jinja_env.filters['timesince'] = timesince
# app.jinja_env.filters['timeuntil'] = timeuntil
# app.jinja_env.filters['jsonencode'] = jsonencode
# app.jinja_env.globals['newrelic_head'] = newrelic_head
# app.jinja_env.globals['newrelic_foot'] = newrelic_foot
# if not app.config.get('TEMPLATE_DEBUG', True):
# compiled_templates = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'compiled_templates')
# compiled_files = path.path(compiled_templates).files()
# if len(compiled_files) <= 1:
# app.jinja_env.compile_templates(compiled_templates, zip=None, py_compile=True)
# app.jinja_env.loader = ModuleLoader(compiled_templates)
configure_logging(app)
return app
def test_document_history():
@save_history
class Book(Document):
title = StringField()
caption = StringField()
establish_mongo_connection()
book = Book()
title_string = "Mother Night"
book.title = title_string
book.save()
history = book.history
eq_(len(history), 1)
eq_(history[0]["changes"]["title"], title_string)
caption_string = "We must be careful about what we pretend to be."
book.caption = caption_string
book.save()
history = book.history
eq_(len(history), 2)
eq_(history[1]["changes"]["caption"], caption_string)
# Clear database
db = _get_db()
collection_names = [c for c in db.collection_names() if not c.startswith("system.")]
for collection in collection_names:
db.drop_collection(collection)
def mongodb(collection=None):
from mongoengine.connection import _get_db
db = _get_db()
if collection is not None:
return getattr(db, collection)
else:
return db
def factory(cls, id=None, type=None, domain=None, uri=None):
if id is None and type is None and (domain is None and uri is None):
return None
if id or (domain and uri):
query = {"_id": ObjectId(id)}
if domain:
query["D"] = domain
if domain and uri:
query["U"] = uri
del query["_id"]
values = _get_db().contribution.find_one(query)
if values is None:
return None
values.setdefault("type")
type = values["type"]
elif type:
values = {}
if type == "html":
return HtmlContribution._from_son(values)
elif type == "markdown":
return MarkdownContribution._from_son(values)
else:
return Contribution._from_son(values)
def tearDown(self):
elapsed = time.time() - self._started_at
print('{} ({}s)'.format(self.id(), round(elapsed, 2)))
db = _get_db()
db.client.drop_database(db)
db.client.close()
self.app_context.pop()
def get_jobs_vs_javascript_libraries():
pipeline = [
{"$redact": {
"$cond": {
"if": {"$eq": ["$deprecated", False]},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
{"$project": {
"_id": 0,
"keywords": 1,
"deprecated": 1
}},
{"$unwind": "$keywords"},
{"$match": {"keywords.types": {"$in": [keyword_type.types["JS_LIB"]]}}},
{"$group": {
"_id": {
"keyword": "$keywords.keyword",
"types": "$keywords.types"
},
"count": {
"$sum": 1
}
}}
]
db = connection._get_db(reconnect=False)
return db.job.aggregate(pipeline)
def get_jobs_vs_programming_languages():
pipeline = [
{"$redact": {
"$cond": {
"if": {"$eq": ["$deprecated", False]},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
{"$project": {
"_id": 0,
"keywords": 1,
"deprecated": 1
}},
{"$unwind": "$keywords"},
{"$match": {"keywords.types": {"$in": [keyword_type.types["PROGRAMMING_LANG"]]}}},
{"$group": {
"_id": {
"keyword": "$keywords.keyword",
"types": "$keywords.types"
},
"count": {
"$sum": 1
}
}}
]
db = connection._get_db(reconnect=False)
return db.job.aggregate(pipeline)
def factory(cls, id=None, type=None, domain=None, uri=None):
if id is None and type is None and (domain is None and uri is None):
return None
if id or (domain and uri):
query = {'_id': pymongo.objectid.ObjectId(id)}
if domain:
query['D'] = domain
if domain and uri:
query['U'] = uri
del query['_id']
values = _get_db().contribution.find_one(query)
if values is None:
return None
values.setdefault('type')
type = values['type']
elif type:
values = {}
if type == 'html':
return HtmlContribution._from_son(values)
elif type == 'markdown':
return MarkdownContribution._from_son(values)
else:
return Contribution._from_son(values)
def factory(cls, id=None, type=None, domain=None, uri=None):
if id is None and type is None and (domain is None and uri is None):
return None
if id or (domain and uri):
query = {'_id': pymongo.objectid.ObjectId(id)}
if domain:
query['D'] = domain
if domain and uri:
query['U'] = uri
del query['_id']
values = _get_db().contribution.find_one(query)
if values is None:
return None
values.setdefault('type')
type = values['type']
elif type:
values = {}
if type == 'html':
return HtmlContribution._from_son(values)
elif type == 'markdown':
return MarkdownContribution._from_son(values)
else:
return Contribution._from_son(values)
def save_draft(cls, user, contrib):
''' @param user: required
@param contrib: ignoring id if blank; always use text
'''
main_vars = {'a': unicode(user._id)}
# Only set contribution ID if ID not blank
if hasattr(contrib, '_id') and contrib._id != '':
main_vars['c'] = unicode(contrib._id)
_get_db().draft.update(
main_vars,
{'$set': {
'txt': contrib.get_text(),
't': contrib.title
}},
upsert=True)
def _get_unique_filename(name):
fs = gridfs.GridFS(_get_db())
file_root, file_ext = os.path.splitext(name)
count = itertools.count(1)
while fs.exists(filename=name):
# file_ext includes the dot.
name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext))
return name
def _get_unique_filename(name):
fs = gridfs.GridFS(_get_db())
file_root, file_ext = os.path.splitext(name)
count = itertools.count(1)
while fs.exists(filename=name):
# file_ext includes the dot.
name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext))
return name
def _clean(self):
#from mongoengine.connection import _get_db
#db = _get_db()
#db.drop_collection("example")
clear_document_registry()
connect(db='mongoenginetest')
self.db = _get_db()
self.db.drop_collection("example")
def teardown_method(self, method):
""" teardown any state that was previously setup with a setup_method
call.
"""
db = _get_db()
db.drop_collection('friends')
db.drop_collection('pending_request')
disconnect(alias='test')
def _clean(self):
#from mongoengine.connection import _get_db
#db = _get_db()
#db.drop_collection("example")
clear_document_registry()
connect(db='mongoenginetest')
self.db = _get_db()
self.db.drop_collection("example")
def setUp(self):
connect(db='mongoenginetest')
self.db = _get_db()
class Person(Document):
name = StringField()
age = IntField()
self.Person = Person
def teardown_method(self, method):
""" teardown any state that was previously setup with a setup_method
call.
"""
db = _get_db()
db.drop_collection('user')
db.drop_collection('invalid_token')
db.drop_collection('reset_password_code')
disconnect(alias='test')
def setUp(self):
connect()
self.db = _get_db('test')
@database('test')
class Person(Document):
name = StringField()
age = IntField()
self.Person = Person
def setUp(self):
connect()
self.db = _get_db()
class Person(Document):
name = StringField()
age = IntField()
uid = ObjectIdField()
self.Person = Person
def __init__(self, *args, **kwargs):
"""
We need to patch jinja_env loader to include flaskext.mongoengine
templates folder.
"""
super(MongoenginePanel, self).__init__(*args, **kwargs)
self.jinja_env.loader = ChoiceLoader([self.jinja_env.loader,
PackageLoader('flask_mongoengine', 'templates')])
self.db = _get_db()
def client(self):
app = create_app(TestConfig)
db = _get_db()
app_context = app.app_context()
app_context.push()
client = app.test_client()
yield client
app_context.pop()
db.client.drop_database(db)
def __init__(self, *args, **kwargs):
"""
We need to patch jinja_env loader to include flaskext.mongoengine
templates folder.
"""
super(MongoenginePanel, self).__init__(*args, **kwargs)
self.jinja_env.loader = ChoiceLoader([self.jinja_env.loader,
PackageLoader('flaskext.mongoengine', 'templates')])
self.db = _get_db()
def __init__(self, level=logging.DEBUG):
""" Constructor
Keyword Arguments:
level -- The logging level
"""
self.log.setLevel(level)
self.ianatags = self.danatags + " IANA "
self.fs = GridFS(_get_db())
def test_connect(self):
"""Ensure that the connect() method works properly.
"""
connect('mongoenginetest')
conn = _get_connection()
self.assertTrue(isinstance(conn, pymongo.connection.Connection))
db = _get_db()
self.assertTrue(isinstance(db, pymongo.database.Database))
self.assertEqual(db.name, 'mongoenginetest')
def setUp(self):
connect()
self.db = _get_db()
class Person(Document):
name = StringField()
age = IntField()
uid = ObjectIdField()
friends = ListField(StringField())
self.Person = Person
def setUp(self):
warnings.simplefilter('always', category=UserWarning)
connect()
self.db = _get_db()
mongoengine.base._document_registry = {}
class Person(Document):
name = StringField()
age = IntField(default=20)
self.person_cls = Person
def test_register_connection(self):
"""Ensure that connections with different aliases may be registered.
"""
register_connection('testdb', 'mongoenginetest2')
self.assertRaises(ConnectionError, _get_connection)
conn = _get_connection('testdb')
self.assertTrue(isinstance(conn, pymongo.connection.Connection))
db = _get_db('testdb')
self.assertTrue(isinstance(db, pymongo.database.Database))
self.assertEqual(db.name, 'mongoenginetest2')
def test_encrypted_field(self):
col = connection._get_db().secret
# Test creating password
s = Secret.objects.create(password='******')
self.assertEqual(s.password, 'hello')
s.reload()
self.assertEqual(s.password, 'hello')
cipher = col.find({'_id': s.id})[0]['password']
self.assertTrue('hello' not in cipher)
self.assertTrue(len(cipher) > 16)
# Test changing password
s.password = '******'
s.save()
s.reload()
self.assertEqual(s.password, 'other')
other_cipher = col.find({'_id': s.id})[0]['password']
self.assertTrue('other' not in other_cipher)
self.assertTrue(len(other_cipher) > 16)
self.assertNotEqual(other_cipher, cipher)
# Make sure password is encrypted differently if we resave.
s.password = '******'
s.save()
s.reload()
self.assertEqual(s.password, 'hello')
new_cipher = col.find({'_id': s.id})[0]['password']
self.assertTrue('hello' not in new_cipher)
self.assertTrue(len(new_cipher) > 16)
self.assertNotEqual(new_cipher, cipher)
self.assertNotEqual(other_cipher, cipher)
# Test empty password
s.password = None
s.save()
s.reload()
self.assertEqual(s.password, None)
raw = col.find({'_id': s.id})[0]
self.assertTrue('password' not in raw)
# Test passwords of various lengths
for pw_len in range(1, 50):
pw = ''.join(random.choice(string.ascii_letters + string.digits) for x in range(pw_len))
s = Secret(password=pw)
s.save()
s.reload()
self.assertEqual(s.password, pw)
def find_events(collection_name, course_id):
results = _get_db()[collection_name].aggregate([{
"$match": {
'context.course_id': course_id
}
}, {
"$group": {
"_id": "$username",
"events": {
"$addToSet": "$event_type"
}
}
}])
return results
def setup(self):
mongoengine.register_connection(mongoengine.DEFAULT_CONNECTION_NAME, TEST_DB_NAME, DB_HOST, DB_PORT)
# TODO
# this code is using pymongo
# maybe mongoengine have the same code
db = _get_db()
for name in db.collection_names(False):
db[name].remove()
self.cover = tempfile.TemporaryFile()
self.cover.write('cover')
self.cover.seek(0)
self.audio = tempfile.TemporaryFile()
self.audio.write('audio')
self.audio.seek(0)
def setup(self):
mongoengine.register_connection(mongoengine.DEFAULT_CONNECTION_NAME,
TEST_DB_NAME, DB_HOST, DB_PORT)
# TODO
# this code is using pymongo
# maybe mongoengine have the same code
db = _get_db()
for name in db.collection_names(False):
db[name].remove()
self.cover = tempfile.TemporaryFile()
self.cover.write('cover')
self.cover.seek(0)
self.audio = tempfile.TemporaryFile()
self.audio.write('audio')
self.audio.seek(0)
def loadFixtures():
from dummy import DummyFixtures
from locations import LocationsFixtures
from mongoengine.connection import _get_db
from flask import current_app
# Clear Database before createing data in it
#db_fd, current_app.config['DATABASE'] = tempfile.mkstemp()
#current_app.config['TESTING'] = True
#client = current_app.test_client()
db = _get_db()
db.connection.drop_database(current_app.config['MONGODB_SETTINGS']['DB'])
fixtures = [DummyFixtures(), LocationsFixtures()]
for fixture in sorted(fixtures, key=lambda f: f.order):
fixture.load()
def countcountryenrolments(collectionname, course_id):
#DO db.clickstream.ensureIndex({country:1})
#DO db.clickstream.ensureIndex( {event_type: 1,"context.course_id": 1} )
#DO db.clickstream.ensureIndex({"context.course_id": 1})
#db.clickstream_tmp.aggregate( [ { $match : {'context.course_id':'UQx/Think101x/1T2014','event_type':'edx.course.enrollment.activated'} }, { $group : { _id : "$country", thecount: { $sum: 1 } } } ] )
results = _get_db()[collectionname].aggregate([ {"$match":{'context.course_id':course_id,
'event_type':'edx.course.enrollment.activated'}},{"$group":{"_id": "$country", "thecount": { "$sum": 1 } }} ])
results = results['result']
countries = []
total = 0
for result in results:
total += result['thecount']
for result in results:
country = {'country': result['_id'], 'count': result['thecount'], 'percentage': (float(result['thecount'])*100/float(total))}
countries.append(country)
return countries
def countfield(field, withpercentages=False, collectionname="log"):
total = 0
reducer = Code("""
function(cur, result) { result.count += 1 }
""")
results = _get_db()[collectionname].group(key={field: 1}, condition={},
initial={"count": 0}, reduce=reducer)
if withpercentages:
for result in results:
total += result['count']
for result in results:
result['name'] = result[field]
if withpercentages:
result['percentage'] = float(float(result['count'])/float(total))*100
del result[field]
return results
def get_jobs_vs_programs():
pipeline = [
{"$redact": {
"$cond": {
"if": {"$eq": ["$deprecated", False]},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
{"$project": {"_id": 0, "programs": 1}},
{"$unwind": "$programs"},
{"$group": {"_id": {
"program": "$programs"
}, "count": {"$sum": 1}}}
]
db = connection._get_db(reconnect=False)
return db.job.aggregate(pipeline)
def countfield(field, withpercentages=False, collectionname="log"):
total = 0
reducer = Code("""
function(cur, result) { result.count += 1 }
""")
results = _get_db()[collectionname].group(key={field: 1},
condition={},
initial={"count": 0},
reduce=reducer)
if withpercentages:
for result in results:
total += result['count']
for result in results:
result['name'] = result[field]
if withpercentages:
result['percentage'] = float(
float(result['count']) / float(total)) * 100
del result[field]
return results
def create_db():
print("Creating db")
db_test_name = os.environ["MONGODB_DB"] + "_test"
db_host = os.environ["MONGODB_HOST"]
# there's no need to drop the database prior if unhealthy
# since it's dropped automatically after tests are run
# TESTING flag disables error catching during request handling,
# so that you get better error reports when performing test requests
# against the application.
yield {
"MONGODB_HOST": db_host,
"MONGODB_DB": db_test_name,
"TESTING": True,
}
print("Destroying db")
db = _get_db()
db.client.drop_database(db)
def get_jobs_vs_locations():
pipeline = [
{"$redact": {
"$cond": {
"if": {"$eq": ["$deprecated", False]},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
{"$project": {"_id": 0, "location": 1}},
{"$unwind": "$location"},
{"$group": {"_id": {
"location": "$location.name",
"longitude": "$location.longitude",
"latitude": "$location.latitude"
}}}
]
db = connection._get_db(reconnect=False)
return db.job.aggregate(pipeline)
def importFile(filePath):
f = open(filePath, "r", encoding="ISO-8859-1")
db = _get_db()
if 'ipToCountry_tmp' in db.collection_names(False):
db.drop_collection('ipToCountry_tmp')
i = 0
for line in f:
if line[0] == '#':
continue
r = list(csv.reader([line]))[0]
doc = IpToCountryTmp(
ipFrom=int(r[0]),
ipTo=int(r[1]),
registry=r[2],
assigned=time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(int(r[3]))),
ctry=r[4],
cntry=r[5],
country=r[6],
)
doc.save()
# if i==10:
# break
# i += 1
f.close()
if 'ipToCountry_bak' in db.collection_names(False):
db.drop_collection('ipToCountry_bak')
if 'ipToCountry' in db.collection_names(False):
db["ipToCountry"].rename("ipToCountry_bak")
db["ipToCountry_tmp"].rename("ipToCountry")
def default(self, value):
"""Convert niche data types, and ``Document`` subclasses.
Conversion notes:
- ``datetime.date`` and ``datetime.datetime`` objects are
converted into UNIX timestamps.
- ``pymongo.objectid.ObjectId`` objects are converted into
``unicode`` strings.
- ``mongoengine.document.Document`` instances are converted
into ``dict`` representations via instance._data.
- ``DBRef`` objects are automagically dereferenced.
"""
if isinstance(value, datetime):
return mktime(value.timetuple())
elif isinstance(value, date):
dt = datetime(value.year, value.month, value.day, 0, 0, 0)
return mktime(dt.timetuple())
elif isinstance(value, ObjectId):
return unicode(value)
elif isinstance(value, (Cursor, QuerySet)):
return list(value)
elif issubclass(value.__class__, (Document, EmbeddedDocument)):
present_keys = dict(filter(lambda k: k[0] in value._present_fields, value.__dict__["_data"].iteritems()))
return present_keys
elif isinstance(value, DBRef):
# dereference
obj = _get_db().dereference(value)
if obj:
del obj["_types"]
return obj
elif isinstance(value, LazyReference):
return value.dereference()
else:
# use no special encoding and hope for the best
return value
def eventtypes(collectionname, course_id):
results = _get_db()[collectionname].aggregate([{
"$match": {
'context.course_id': course_id
}
}, {
"$group": {
"_id": {
"type": "$event_type",
"date": {
"$concat": [{
"$substr": [{
"$year": ['$time_date']
}, 0, 4]
}, "_", {
"$substr": [{
"$week": ['$time_date']
}, 0, 2]
}]
}
},
"users": {
"$addToSet": "$username"
}
}
}, {
"$unwind": "$users"
}, {
"$group": {
"_id": "$_id",
"userCount": {
"$sum": 1
}
}
}])
results = results['result']
return results
def tearDown(self):
db = _get_db()
db.client.drop_database(db)
def tearDown(self):
db = _get_db()
collections = db.collection_names(include_system_collections=False)
for collection in collections:
db[collection].drop()
def tearDownClass(cls):
db = _get_db()
db.client.drop_database(db)
def setUp(self):
connect(db='mongoenginetest')
self.db = _get_db()
def tearDown(self):
db = _get_db()
db.drop_collection("todo")
