def main():
for collection in collections:
print "RUN %s" % collection
if collection == "tempentities":
print "PASS"
if collection == "logs":
print "PASS"
else:
ret = mongoExportImport(collection)
if 0 == ret:
print "COMPLETE"
else:
print "ERROR restoring collection '%s'" % collection
print
convertEntities()
try:
utils.runMongoCommand('db.runCommand( {createCollection:"logs", capped:true, size:500000} )')
except:
utils.printException()
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--drop', action="store_true", default=False,
help="drop existing collections before importing")
parser.add_argument("-s", "--source", default=None, type=str, help="db to import from")
parser.add_argument("-t", "--target", default=None, type=str, help="db to import to")
args = parser.parse_args()
host, port = utils.get_db_config(args.source)
utils.log("SOURCE: %s:%s" % (host, port))
old_host = host
old_connection = pymongo.Connection(host, port)
old_database = old_connection['stamped']
collections = old_database.collection_names()
new_host = args.target
if new_host is None:
dest = MongoDBConfig.getInstance()
new_host = dest.host
utils.log("DEST: %s:%s" % (new_host, port))
if not os.path.isdir('/stamped/tmp/stamped/'):
os.makedirs('/stamped/tmp/stamped')
ignore = set([
'tempentities', 'logs', 'logstats',
])
for collection in collections:
print 'RUN %s' % collection
if collection in ignore:
print 'PASS'
else:
ret = mongoExportImport(collection, old_host, new_host)
if 0 == ret:
print 'COMPLETE'
else:
print "ERROR restoring collection '%s'" % collection
print
try:
utils.runMongoCommand('db.runCommand( {createCollection:"logs", capped:true, size:500000} )')
except:
utils.printException()
def main():
old_connection = pymongo.Connection(OLD_HOST, 27017)
new_connection = pymongo.Connection(NEW_HOST, 27017)
old_database = old_connection['stamped']
new_database = new_connection['stamped']
collections = old_database.collection_names()
if not os.path.isdir('/stamped/tmp/stamped/'):
os.makedirs('/stamped/tmp/stamped')
ignore = set([
'tempentities', 'logs', 'logstats',
])
for collection in collections:
print 'RUN %s' % collection
if collection in ignore:
print 'PASS'
else:
ret = mongoExportImport(collection)
if 0 == ret:
print 'COMPLETE'
else:
print "ERROR restoring collection '%s'" % collection
print
convertEntities()
try:
utils.runMongoCommand('db.runCommand( {createCollection:"logs", capped:true, size:500000} )')
except:
utils.printException()
def getStats(self, store=False):
unique_user_stats = {}
# subcategory_stats = { }
# source_stats = { }
# find the number of unique, active users from the past day / week / month
cmd = "db.logstats.group({ reduce: function(obj,prev) { prev.users[obj.uid] = 1; }, cond:{uid: {$exists: true}, %s}, initial: {users:{}, }, key:{}, })"
times = [
("one_day", 24 * 60 * 60000), # 24 hours in milliseconds
("one_week", 168 * 60 * 60000), # 7 days in milliseconds
("one_month", 720 * 60 * 60000), # 30 days in milliseconds
("total", None),
]
for k, v in times:
try:
if v is not None:
mongo_cmd = cmd % ("bgn: {$gte: new Date(new Date() - %s)}, " % v)
else:
mongo_cmd = cmd % ""
ret = utils.runMongoCommand(mongo_cmd)
num_users = len(ret[0]["users"])
# print "%s) %d" % (k, num_users)
unique_user_stats[k] = num_users
except:
utils.printException()
# evaluate distribution of comments per stamp
cmd = "db.comments.group({ reduce: function(obj,prev) { prev.count += 1 }, initial: {count: 0, }, key:{stamp_id:1}, })"
ret = utils.runMongoCommand(cmd)
num_comments_per_stamp = utils.get_basic_stats(ret, "count")
# evaluate distribution of comments per user
cmd = "db.comments.group({ reduce: function(obj,prev) { prev.count += 1 }, initial: {count: 0, }, key:{user:1}, })"
ret = utils.runMongoCommand(cmd)
num_comments_per_user = utils.get_basic_stats(ret, "count")
custom_stats = {}
cmds = {
# num stamps per user
"num_stamps_per_user": (
"db.stamps.group({ reduce: function(obj,prev) { prev.count += 1 }, %sinitial: {count: 0, }, key:{user:1}, })",
1,
),
# num stamps left per user
"num_stamps_left_per_user": (
'db.users.group({ reduce: function(obj,prev) { prev.count = obj.stats.num_stamps_left }, %sinitial: {count: 0, }, key:{_id:1}, finalize: function(obj) { return { "count" : obj.count }}})',
0,
),
# num likes per stamp
"num_likes_per_stamp": (
'db.stamps.group({ reduce: function(obj,prev) { if (obj.stats.hasOwnProperty("num_likes")) { prev.count = obj.stats.num_likes; } else { prev.count = 0; } }, %sinitial: {count: 0, }, key:{_id:1}, finalize: function(obj) { return { "count" : obj.count }}})',
2,
),
# num likes per user
"num_likes_per_user": (
'db.users.group({ reduce: function(obj,prev) { if (obj.stats.hasOwnProperty("num_likes")) { prev.count = obj.stats.num_likes; } else { prev.count = 0; } }, %sinitial: {count: 0, }, key:{_id:1}, finalize: function(obj) { return { "count" : obj.count }}})',
0,
),
# num followers per user
"num_followers_per_user": (
'db.users.group({ reduce: function(obj,prev) { if (obj.stats.hasOwnProperty("num_followers")) { prev.count = obj.stats.num_followers; } else { prev.count = 0; } }, %sinitial: {count: 0, }, key:{_id:1}, finalize: function(obj) { return { "count" : obj.count }}})',
0,
),
}
for cmd_key, cmd in cmds.iteritems():
custom_stats[cmd_key] = {}
for k, v in times:
if v is not None:
if cmd[1] == 0:
continue
elif cmd[1] == 1:
field = "timestamp.created"
else:
field = "timestamp.modified"
mongo_cmd = cmd[0] % ('cond: { "%s": {$gte: new Date(new Date() - %s)} }, ' % (field, v))
else:
mongo_cmd = cmd[0] % ""
ret = utils.runMongoCommand(mongo_cmd)
stats = utils.get_basic_stats(ret, "count")
custom_stats[cmd_key][k] = stats
# TODO: incorporate more metrics
# https://docs.google.com/a/stamped.com/spreadsheet/ccc?key=0AmEQSQLwlDtTdHoweWRZSjhXTm5Xb3NvSFBCQ0szWlE&hl=en_US#gid=0
"""
for source in EntitySources()._elements:
count = self._entityDB._collection.find({"sources.%s" % source : { "$exists" : True }}).count()
source_stats[source] = count
for subcategory in subcategories:
count = self._entityDB._collection.find({"subcategory" : subcategory}).count()
subcategory_stats[subcategory] = count
"""
stats = {
"entities": {"count": self._entityDB._collection.count()},
"users": {"count": self._userDB._collection.count(), "active": unique_user_stats},
"comments": {
"count": self._commentDB._collection.count(),
"per_stamp": num_comments_per_stamp,
"per_user": num_comments_per_user,
},
"likes": {"per_stamp": custom_stats["num_likes_per_stamp"], "per_user": custom_stats["num_likes_per_user"]},
"stamps": {
"count": self._stampDB._collection.count(),
"per_user": custom_stats["num_stamps_per_user"],
"left_per_user": custom_stats["num_stamps_left_per_user"],
},
}
# optionally store stats
def _store(prefix, stats):
for k, v in stats.iteritems():
key = "%s.%s" % (prefix, k)
if isinstance(v, dict):
_store(key, v)
else:
if isinstance(v, basestring):
try:
v = float(v)
except:
continue
self._statsSink.time(key, v)
if store:
_store("stamped.stats", stats)
return stats