def calc_rating_price_frequency(sCol):
sFuncMap = bson.Code("""
// BANDWIDTH is defined externally by 'scope'
function(){
// Count documents grouped by rating + rounded price
var iPrice = Math.ceil(this.price / BANDWIDTH) * BANDWIDTH;
emit(
[this.rating, iPrice], // Key (used for grouping)
1 // Values appended to data of group
);
}
""",
scope={'BANDWIDTH': 500})
sFuncReduce = bson.Code("""
function(lKey, lData){
// E.g. lKey = [100, 2500]; // Rating, price
// E.g. lData = [1,1,1,1,1,1,1,1]; // One item per doc
var iTotal = 0;
for (var k in lData){ iTotal += lData[k]; }
return iTotal
}
""",
scope={})
sFuncFinalize = bson.Code("""
function(lKey, iReducedValue){
// E.g. lKey = [100, 2500]; // Rating, price
// E.g. iReducedValue = 22; // Group document count
return {'rating':lKey[0],
'price':lKey[1],
'count': iReducedValue
}
}
""",
scope={})
dResponse = sCol.map_reduce(sFuncMap,
sFuncReduce,
finalize=sFuncFinalize,
out={'inline': 1})
lResults = []
for dDoc in dResponse['results']:
lResults += [dDoc['value']]
return lResults
def test_map_reduce(self):
# Count number of documents with even and odd _id
yield from self.make_test_data()
expected_result = [{'_id': 0, 'value': 100}, {'_id': 1, 'value': 100}]
map_fn = bson.Code('function map() { emit(this._id % 2, 1); }')
reduce_fn = bson.Code('''
function reduce(key, values) {
r = 0;
values.forEach(function(value) { r += value; });
return r;
}''')
yield from self.db.tmp_mr.drop()
# First do a standard mapreduce, should return AsyncIOMotorCollection
collection = self.collection
tmp_mr = yield from collection.map_reduce(map_fn, reduce_fn, 'tmp_mr')
self.assertTrue(
isinstance(tmp_mr, motor_asyncio.AsyncIOMotorCollection),
'map_reduce should return AsyncIOMotorCollection, not %s' % tmp_mr)
result = yield from tmp_mr.find().sort([('_id', 1)
]).to_list(length=1000)
self.assertEqual(expected_result, result)
# Standard mapreduce with full response
yield from self.db.tmp_mr.drop()
response = yield from collection.map_reduce(map_fn,
reduce_fn,
'tmp_mr',
full_response=True)
self.assertTrue(isinstance(response, dict),
'map_reduce should return dict, not %s' % response)
self.assertEqual('tmp_mr', response['result'])
result = yield from tmp_mr.find().sort([('_id', 1)
]).to_list(length=1000)
self.assertEqual(expected_result, result)
# Inline mapreduce
yield from self.db.tmp_mr.drop()
result = yield from collection.inline_map_reduce(map_fn, reduce_fn)
result.sort(key=lambda doc: doc['_id'])
self.assertEqual(expected_result, result)
def find_distinct_specs(sCol):
sFuncMap = bson.Code("""
function (){
// 'this' refers to the current document
for(var key in this.spec){
emit(key, null);
}
}
""")
sFuncReduce = bson.Code("""
function(mKey, lValues){
print ('Reducing'+mKey+lValues);
return null;
}
""")
dResponse = {}
dResponse = sCol.map_reduce(sFuncMap, sFuncReduce, out={'inline': 1})
lKeys = [x['_id'] for x in dResponse['results']]
lKeys = [x.replace('.', '.') for x in lKeys]
return lKeys
"""
client = pymongo.MongoClient()
db = client.newYork
#subway = db.subway
#dataSubway = requests.get("https://data.cityofnewyork.us/resource/he7q-3hwy.json").json()
#resSubway = subway.insert_many(dataSubway)
park = db.park
#dataPark = requests.get("https://data.cityofnewyork.us/resource/y6ja-fw4f.json").json()
#resPark = park.insert_many(dataPark)
school = db.school
#dataSchool = requests.get("https://data.cityofnewyork.us/resource/8pnn-kkif.json").json()
#resSchool = school.insert_many(dataSchool)
db.school.find().forEach(bson.Code('''
function(p) {
print(p.name)
}'''))
#print(park.find())
#def getParkLePlusProche(longit, latit)
# return 30;
#fonction ->
# param position school
# renvoie le park le plus proche de school
def code(js):
"""Convert Javascript string to binary format that PyMongo requires."""
return bson.Code(js)
def mapreduce_user_registration_types():
map = bson.Code(map_js)
reduce = bson.Code(reduce_js)
result = mongo_db['UserData'].map_reduce(map, reduce, "registration_types")
for doc in result.find():
print doc
def daily_problem_count():
result = mongo_db['ProblemLog'].map_reduce(bson.Code(map),
bson.Code(reduce),
"daily_problem_count")
def mapreduce_videos_per_user():
result = mongo_db['UserVideo'].map_reduce(bson.Code(map_js),
bson.Code(reduce_js),
"videos_per_user")
def mapreduce_exercises_per_user():
result = mongo_db['UserExercise'].map_reduce(bson.Code(map_js),
bson.Code(reduce_js),
"exercises_per_user")
import datetime
from cStringIO import StringIO
import pymongo
from framework.mongo import database
from website import models
from website.app import app, init_app
from scripts.analytics import utils
from scripts.analytics import settings
mapper = bson.Code('''function() {
emit(this.action, 1);
}''')
reducer = bson.Code('''function(key, values) {
var count = 0;
for (var i=0; i<values.length; i++) {
count += values[i];
}
return count;
}''')
out = {'replace': settings.TABULATE_LOGS_RESULTS_COLLECTION}
if (choice == 1):
insert_function(items_collection)
elif (choice == 2):
delete_function(items_collection)
elif (choice == 3):
update_function(items_collection)
elif (choice == 4):
ch = int(
input(
'\n1. Number of items per brand\t2. Number of items per category\t3.Exit\nEnter your choice:'
))
if (ch == 1):
cat = input('Enter the category: ')
regex_brand = '.*' + cat + '.*'
map = bson.Code("""function(){
emit(this.brand,1);
}""")
reduce = bson.Code(
"function (key, values) {"
" var total = 0;"
" for (var i = 0; i < values.length; i++) {"
" total += values[i];"
" }"
" return total;"
"}")
#query_main
val = items_collection.map_reduce(map,
reduce,
out='output',
query={
'main_cat': {
'$project': {
'state': 1,
'_id': 0
}
}, {
'$group': {
'_id': '$state',
'count': {
'$sum': 1
}
}
}, {
'$sort': {
'count': -1
}
}, {
'$limit': 5
}])
for r in result:
print(r)
###run mapreduce
mapper = bson.Code('''function() {emit(this.state, 1)}''')
reducer = bson.Code('''function(key,values){return Array.sum(values)}''')
db.postalCodes.map_reduce(map=mapper, reduce=reducer, out='pymr_out')
c = db.pymr_out.find(sort=[('value', pymongo.DESCENDING)], limit=5)
for elem in c:
print(elem)
data = bson._dict_to_bson(
{"a": value},
True, # check_keys
bson.DEFAULT_CODEC_OPTIONS)
with self.assertRaises(bsonnumpy.error) as context:
bsonnumpy.sequence_to_ndarray([data], np.dtype([("a", dtype)]), 1)
self.assertIn("unsupported BSON type: %s" % type_name,
str(context.exception))
return test
for value_, type_name_, dtype_ in [
(bson.Code(""), "Code", "V10"),
(bson.Code("", {'scope': 1}), "Code with Scope", "V10"),
({}, "Sub-document", "V10"),
(bson.MinKey(), "MinKey", "V10"),
(bson.MaxKey(), "MaxKey", "V10"),
(bson.regex.Regex("pattern"), "Regular Expression", "V10"),
(bson.timestamp.Timestamp(0, 0), "Timestamp", "V10"),
(None, "Null", "V10"),
]:
test_name = "test_unsupported_%s" % type_name_.lower()
setattr(TestFromBSONScalars, test_name,
_make_test_fn(value_, type_name_, dtype_))
if __name__ == "__main__":
unittest.main()
