def get_db(db_name):
import pymongo
DB_HOST = ["localhost"]
DB_PORT = 27017
db = pymongo.Connection(DB_HOST, DB_PORT)[db_name]
return db
- Split existing texts
- Rewrites existing links
- Rewrites history
"""
import sys
import os
import re
from pprint import pprint
import pymongo
p = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, p)
sys.path.insert(0, p + "/sefaria")
from sefaria.texts import *
connection = pymongo.Connection()
db = connection[SEFARIA_DB]
db.authenticate(SEFARIA_DB_USER, SEFARIA_DB_PASSWORD)
# Insert index records from JSON
mishneh_torah = json.loads(open("mishneh-torah.json", "r").read())
for topic in mishneh_torah:
db.index.save(topic)
# Build the mapping of numbered sections to new text names
index = []
indices = db.index.find({"categories": "Mishneh Torah"}).sort([["order.0", 1]])
section = -1
import pymongo # Setup Mongo conn = pymongo.Connection() # defaults to localhost db = conn.agile_data emails = db['emails'] email_list = emails.find()[0:20] for email in email_list: print email
def log_items_to_person_proposals(start, end, found_cb):
single_map_cb = Code("""
function() {
emit(this.i, {pid: [this.p]});
}
""")
single_reduce_cb = Code("""
function(item, vals) {
var res = {pid:[]};
for(var v in vals) {
var value = vals[v];
if(value.pid !== undefined) {
for(var j in value.pid) {
res.pid.push(value.pid[j]);
}
}
}
return res;
}
""")
date_query = {'d': {"$gte": start, "$lt": end}, 'p': {'$exists': True}}
db_host = config.get('download_log', 'server')
db_port = config.getint('download_log', 'port')
connection = pymongo.Connection(db_host, db_port)
db = connection[config.get('download_log', 'db_name')]
db_name_single_items = config.get('download_log', 'single_collection')
db_name_cart_items = config.get('download_log', 'cart_collection')
db_name_uniq_cart_items = config.get('download_log',
'uniq_cart_collection')
db_name_uniq_items = config.get('download_log', 'uniq_single_collection')
db_name_user2proposals = config.get('download_log',
'user2proposals_collection')
cart_map_cb = Code("""
function() {
emit(this.c, {pid: [this.p]});
}
""")
cart_reduce_cb = Code("""
function(item, vals) {
var res = {pid:[]};
for(var v in vals) {
var value = vals[v];
if(value.pid !== undefined) {
for(var j in value.pid) {
res.pid.push(value.pid[j]);
}
}
}
return res;
}
""")
db[db_name_cart_items].ensure_index([('d', pymongo.ASCENDING)])
db[db_name_cart_items].map_reduce(cart_map_cb,
cart_reduce_cb,
out={'replace': db_name_uniq_cart_items},
query=date_query)
def cart_process(chunk_items, collection):
sql = "SELECT cart_id, item_id FROM myemsl.cart_items WHERE cart_id in (%s) ORDER by cart_id;" % (
','.join([str(i) for i in chunk_items]))
cnx = myemsldb_connect(myemsl_schema_versions=['1.3'])
cursor = cnx.cursor()
cursor.execute(sql)
old_cart_id = None
cart_list = {}
for row in cursoriter(cursor):
toset = cart_list.setdefault(row[0], [])
toset.insert(0, row[1])
for key in cart_list.iterkeys():
collection.update({'_id': key},
{"$set": {
'value.i': cart_list[key]
}},
w=1)
chunk_items = []
collection = db[db_name_uniq_cart_items]
for item in db[db_name_uniq_cart_items].find(fields={'_id': 1}):
chunk_items.insert(0, int(item['_id']))
if len(chunk_items) >= chunk_size:
cart_process(chunk_items, collection)
chunk_items = []
if len(chunk_items) > 0:
cart_process(chunk_items, collection)
db[db_name_single_items].ensure_index([('d', pymongo.ASCENDING)])
db[db_name_single_items].map_reduce(single_map_cb,
single_reduce_cb,
out={'replace': db_name_uniq_items},
query=date_query)
map2_cb = Code("""
function() {
for(var i in this.value.i) {
emit(this.value.i[i], {pid: this.value.pid});
}
}
""")
reduce2_cb = Code("""
function(item, vals) {
var res = {p:[], pid:[]};
for(var v in vals) {
var value = vals[v];
if(value.p !== undefined) {
for(var j in value.p) {
res.p.push(value.p[j]);
}
}
if(value.pid !== undefined) {
for(var j in value.pid) {
res.pid.push(value.pid[j]);
}
}
}
return res;
}
""")
db[db_name_uniq_cart_items].map_reduce(map2_cb,
reduce2_cb,
out={'reduce': db_name_uniq_items})
def proposals_process(chunk_items, collection):
query = {
"fields": ["proposals"],
"query": {
"constant_score": {
"filter": {
"ids": {
"values": chunk_items
}
}
}
}
}
server = config.get('elasticsearch', 'server')
results = json.loads(
call_curl("%s/myemsl_current_simple_items/simple_items/_search" %
(server),
method='POST',
idata=json.dumps(query)))
if results['hits']['total'] != len(chunk_items):
raise Exception('Failed to find some items')
for hit in results['hits']['hits']:
if 'fields' in hit and 'proposals' in hit['fields']:
collection.update(
{'_id': int(hit['_id'])},
{"$set": {
'value.p': hit['fields']['proposals']
}},
w=1)
chunk_items = []
collection = db[db_name_uniq_items]
for item in db[db_name_uniq_items].find(fields={'_id': 1}):
chunk_items.insert(0, int(item['_id']))
if len(chunk_items) >= chunk_size:
proposals_process(chunk_items, collection)
chunk_items = []
if len(chunk_items) > 0:
proposals_process(chunk_items, collection)
map3_cb = Code("""
function() {
for(var i in this.value.pid) {
var pid = this.value.pid[i];
emit(pid, {p: this.value.p});
}
}
""")
reduce3_cb = Code("""
function(item, vals) {
var tp = {}
var res = {p:[]};
for(var v in vals) {
var value = vals[v];
for(var j in value.p) {
tp[value.p[j]] = 1;
}
}
for(var p in tp) {
res.p.push(p);
}
return res;
}
""")
db[db_name_uniq_items].map_reduce(map3_cb,
reduce3_cb,
out={'replace': db_name_user2proposals})
for item in db[db_name_user2proposals].find():
if item['value']['p'] != None:
found_cb(int(item['_id']), item['value']['p'])
def initVdsMongo(self):
self.vdsMongo = pymongo.Connection("bang")['freeway']['vds']
'''
This file includes code to:
Analyze the text content of a tweet object to build a training set using frequency analysis.
Naive Bayes classifier used for sentiment analysis.
'''
import pymongo as mongo
import geo
import simplejson as json
import time
connection = mongo.Connection()
db = connection.CompProb
word_collection = db.words
keep = ['no', 'ok', 'up']
exclude = set([
"2012 election", "presidential election", "presidential debate",
"US president", "obama administration", "president of the united states",
"republican", "ron paul", "mitt romney", "mitt", "romney", "democrat",
"democrats", "democratic", "barack obama", "obama", "barack", "about",
"after", "all", "also", "and", "another", "any", "are", "because", "been",
"before", "being", "between", "both", "came", "come", "could", "each",
"for", "from", "get", "got", "has", "had", "have", "her", "here", "him",
"himself", "his", "how", "into", "like", "make", "many", "might", "more",
"most", "much", "must", "now", "only", "other", "our", "out", "over",
"said", "same", "see", "should", "since", "some", "still", "such", "take",
"than", "that", "the", "their", "them", "then", "there", "these", "they",
"this", "those", "through", "too", "under", "very", "was", "way", "well",
"were", "what", "where", "which", "while", "who", "with", "would", "you",
"your"
def run(server,
db,
coll,
method='find',
query=None,
limit=1000,
fields=None,
sort=None,
fill=None):
# Create an empty response object.
response = tangelo.empty_response()
# Check the requested method.
if method not in ['find', 'insert']:
response['error'] = "Unsupported MongoDB operation '%s'" % (method)
return bson.json_util.dumps(response)
# Decode the query strings into Python objects.
try:
if query is not None: query = decode(query, 'query', response)
if fields is not None: fields = decode(fields, 'fields', response)
if sort is not None: sort = decode(sort, 'sort', response)
if fill is not None:
fill = decode(fill, 'fill', response)
else:
fill = True
except ValueError:
return bson.json_util.dumps(response)
# Cast the limit value to an int.
try:
limit = int(limit)
except ValueError:
response[
'error'] = "Argument 'limit' ('%s') could not be converted to int." % (
limit)
return bson.json_util.dumps(response)
# Create database connection.
try:
c = pymongo.Connection(server)[db][coll]
except pymongo.errors.AutoReconnect:
response['error'] = "Could not connect to MongoDB server '%s'" % (
server)
return bson.json_util.dumps(response)
# Perform the requested action.
if method == 'find':
# Do a find operation with the passed arguments.
it = c.find(spec=query, fields=fields, limit=limit, sort=sort)
# Create a list of the results.
if fill:
results = [x for x in it]
else:
results = []
# Create an object to structure the results.
retobj = {}
retobj['count'] = it.count()
retobj['data'] = results
# Pack the results into the response object, and return it.
response['result'] = retobj
else:
raise RuntimeError("illegal method '%s' in module 'mongo'")
# Return the response object.
return bson.json_util.dumps(response)
import pymongo
import random
from pyokapi import *
import threading
import time
_conn = pymongo.Connection(MONGO_HOST, MONGO_PORT)
_apis = _conn.okapi.service
_deploy = _conn.okapi.deploys
_docs = _conn.okapi.doc
dkrt = "http://172.17.0.1:2375"
def get_all_containers():
return InvokeService(
dkrt + '/containers/json?all=1&filters={"status":["running"]}').get()
def create_container(author, runtime):
name = "%s.%s" % (author, runtime)
if runtime == "python":
image = "ls/okapi-py2"
elif runtime == "python3":
image = "ls/okapi-py3"
elif runtime == "java":
image = "ls/okapi-java"
else:
raise Exception("not supported runtime: %s" % runtime)
d = {
"Hostname": name,
def prepare_db(self):
self.connection = pymongo.Connection(self.args.host, self.args.port)
self.db = self.connection[self.args.database]
#!/bin/python
#run file like this
#execfile('/xchip/cogs/hogstrom/scripts/dose_timeCourse/ts_query.py')
#cd /xchip/cogs/hogstrom/scripts/dose_timeCourse
import pymongo
conn = pymongo.Connection('vitalstatistix')
affogato = conn['affogato']
affogato.authenticate('cmap_user', 'l1000')
signature = affogato['signature']
### QUERY 1 ###
#signature.find({brew_prefix:{$in:[/^DOS/,/^PAC/]}},{sig_id:1}).count()
qCurs = signature.find_one({'det_plate':'DOS059_PC3_24H_X2_F3B4_DUO52HI53LO','is_gold':1.0},{'cell_id':True,'pert_type':True,'sig_id':True,'pert_desc':True,'is_gold':True,'pert_dose':True,'pert_time':True,'cell_id':True})
qLis = []
for x in qCurs:
qLis.append(x)
### QUERY 2 ###
testGene = 'NOTCH2'
qCurs = signature.find({'pert_type':'trt_sh','pert_desc':testGene,'is_gold':1.0},{'cell_id':True,'pert_type':True,'sig_id':True,'pert_desc':True,'is_gold':True,'pert_dose':True,'pert_time':True,'cell_id':True})
qLis = []
for x in qCurs:
qLis.append(x)
sigs = [str(x['sig_id']) for x in qLis]
isGlds = [str(x['is_gold']) for x in qLis]
dose = [str(x['pert_dose']) for x in qLis]
def add_mongo_db(config):
#for use mongodb:
db_url = urlparse(config.registry.settings['mongo_uri'])
conn = pymongo.Connection(host=db_url.hostname)
config.registry.settings['db_conn'] = conn
import pymongo
import time
import math
db = pymongo.Connection('localhost',27017).bbarnes_test
earth = db.universe.find_one({'name' : "Earth"})
x = earth['x_pos']
y = earth['y_pos']
r = earth['distance_from_star']
name = earth['name']
id = earth['_id']
parent_id = earth['parent_id']
parent = db.universe.find_one({ '_id' : parent_id})
star_x = parent['x_pos']
star_y = parent['y_pos']
print "x is ",x
print "y is ",y
print "radius is ",r
print "planet in question is " ,name
print "its starting at " , x,y
def x_increase():
#-*- coding:utf-8 -*-
import sys
import urllib2
import urllib
import cookielib
import hashlib
import Image
import pymongo
from scrapy import Selector
reload(sys)
sys.setdefaultencoding('utf-8')
# initialize mongodb connection
try:
con = pymongo.Connection('127.0.0.1', 27017)
kd_db = con.kd
table = kd_db.info
except Exception as e:
print 'Cannot connect mongodb!'
print e
exit(-1)
def get_html_by_data(url, use_cookie=False):
data = {}
post_data = urllib.urlencode(data)
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
req = urllib2.Request(url)
if use_cookie:
# -*- coding: utf-8 -*-
import pymongo
import os
import sys
import string
import codecs
import time
ISOTIMEFORMAT='%Y-%m-%dT%XZ'
#import math
#connect
c = "昌平镇"
conn = pymongo.Connection("10.214.0.147",27017)
#set database
db = conn.Air
#db1 = conn.Beijing
#db.authenticate("pm","ccntgrid")
collection = db.Stations
#collection_w = db.Weather
#collection_b = db1.BJ_Weather_Forecast
#initialize the tmp result
result = {}
#4 of weather prediction
#select result
print "Ready"
#ofile = codecs.open('X_1', 'w',"utf-8")
ofile1 = codecs.open('station_Changping.txt', 'w',"utf-8")
distinct_time = collection.distinct('time_point')
if len(sys.argv) < 4:
usage_message = """usage: %s db coll f1 [f2] [f3...]
For one or more files containing edx tracking logs, insert into the
collection given. Files ending .gz they are decompressed on the fly.
Files successfully loaded are tracked in coll_incremental. If already
loaded, skip.
"""
sys.stderr.write(usage_message % sys.argv[0])
sys.exit(1)
db_name = sys.argv[1]
collection_name = sys.argv[2]
# Get database connection and collections
connection = pymongo.Connection('localhost', 27017)
db = connection[db_name]
events_coll = db[collection_name]
imported_coll = db[collection_name + "_imported"]
total_error = 0
total_success = 0
# Connect to course_structure database
STRUCTURE_COLLECTION_NAME = 'course_structure'
struct_coll = db[STRUCTURE_COLLECTION_NAME]
# Append course_structure info to record dict
def append_course_struct(id):
try:
try:
logging.info("Connecting to source MongoDB server: ".format(
self.opts.src))
self.mongo_src = pymongo.Connection(host=self.opts.src)
logging.info(
"Connected successfully to source MongoDB server, version: {0}"
.format(self.mongo_src.server_info()['version']))
except pymongo.errors.ConnectionFailure, e:
logging.error(
"Could not connect to source MongoDB server - {0}".format(e))
sys.exit(0)
try:
logging.info("Connecting to destination MongoDB server: ".format(
self.opts.dst))
self.mongo_dst = pymongo.Connection(host=self.opts.dst)
logging.info(
"Connected successfully to destination MongoDB server, version: {0}"
.format(self.mongo_dst.server_info()['version']))
except pymongo.errors.ConnectionFailure, e:
logging.error(
"Could not connect to destination MongoDB server - {0}".format(
e))
sys.exit(0)
def start_sharding(self):
logging.debug("START SHARDING")
self.connect_to_servers()
self.setup_database_shards()
logging.debug("END SHARDING")
import json
from collections import defaultdict
import codecs
import pymongo
client = pymongo.Connection()
db = client.xuetangx
category_stats = db.category_stats
category2courses = defaultdict(list)
course_info = json.load(open('course_info.json'))
course2image = json.load(open('course2image.json'))
for course_id, course in course_info.items():
if course['category'] is None: continue
categories = [x[1] for x in json.loads(course['category']).items()]
for category in categories:
category2courses[category].append({
'course_id':
course_id,
'name':
course['name'],
'image':
course2image.get(course_id, None)
})
category_db = list()
for category, courses in category2courses.items():
category_db.append({'category': category, 'courses': courses})
# coding: utf-8
import json
import os
import pymongo
from warnings import warn
config = {}
try:
try:
config_file = os.environ["POPCORN_CONF"]
except KeyError:
config_file = "popcorn.conf"
with open(config_file) as fh:
config = json.load(fh)
del config_file
except (IOError, ValueError):
warn("I need configuration!")
db = pymongo.Connection()["popcorn"]
try:
_db_config = {"url": "mongodb://127.0.0.1", "name": "popcorn"}
_db_config.update(config.get("database", {}))
db = pymongo.Connection(_db_config["url"])[_db_config["name"]]
del _db_config
except pymongo.errors.AutoReconnect:
warn("I need database!")
db.messages.ensure_index([("channel", 1), ("time", -1)])
data = "".join(str(x) for x in data if str(x).isalpha())
if (timestamp - lastTime) / 1000 > 10:
lastTime = timestamp
lat = 40.48 + (random.random() / 5 - .2)
lon = -74.440 - (random.random() / 5 - .2)
return [{
"timestamp": timestamp,
"user": data,
"lat": lat,
"lon": lon
}, lastTime]
print "Bam,nice swipe " + data + "!"
return [-1, lastTime]
dbConnection = pymongo.Connection("172.31.74.128",
27017) #default port, change if ne
errors = ["block", "thenticat", "response", "AAA", "FF", "ff"]
db = dbConnection.db1
brate = 115200
ser = serial.Serial('/dev/ttyACM0', brate)
while (1):
temp = ser.readline()
a = [x for x in errors if temp.find(x) != -1]
if len(a) == 0:
temp, lastTime = parseData(temp, lastTime)
ser.flushInput()
if temp != -1:
print temp
print "Bam,into the db"
db.dataSet.save(temp)
def filterFunction(jobid,filename,inputPtnPath,model,table,partAns,st,domainRange):
contentPtnJson = json.load(open(os.path.join(inputPtnPath,filename),"r"))
print "Worker %d : Read %s into filter" % (jobid,filename)
connect = pymongo.Connection()
db = connect.projizz
collection = db.result.yago.answer
queries = map(lambda x: x[:-4], contentPtnJson)
itr = collection.find({"revid":{"$in":queries}})
print "worker %d query=%d, result=%d" % (jobid,len(queries),itr.count())
count = 0
for ans in itr:
count += 1
key = "%s.txt" % (ans["revid"])
types = ans["type"]
# Now only consider properties, no references.
relation = ans["properties"]
ptnEx = contentPtnJson[key]
relaEx = []
for line in ptnEx:
for ptn in line[1]:
ptnId = "%d" % (ptn[0])
rfp = table[ptnId]["relations"]
# never seen pattern
if not ptnId in st:
continue
# if only one relation
if len(rfp) < 2:
if st[ptnId][0][1]["support"] > 0 and not rfp[0] in relaEx:
relaEx.append(rfp[0])
# more than one relation
else:
# using the first as the answer
if st[ptnId][0][1]["support"] > 0 and not rfp[0] in relaEx:
relaEx.append(rfp[0])
# Remove impossible relations
toBeRemove = []
for attribute in relaEx:
# speical case, produced
if domainRange[attribute] == "":
continue
if not domainRange[attribute]["domain"] in types:
if not attribute in toBeRemove:
toBeRemove.append(attribute)
for attribute in toBeRemove:
relaEx.remove(attribute)
# Evaluation
for attribute in partAns:
postive = False
true = False
if attribute in relaEx:
postive = True
if attribute in relation:
true = True
if true:
if postive:
partAns[attribute]["tp"].append(ans["revid"])
else:
partAns[attribute]["fn"].append(ans["revid"])
else:
if postive:
partAns[attribute]["fp"].append(ans["revid"])
else:
partAns[attribute]["tn"].append(ans["revid"])
if count % 100 == 0:
print "worker #%d done %d." % (jobid,count)
return partAns
import pymongo # setup: sudo pip install pymongo
from ConfigParser import ConfigParser
import os.path
import sys
conf_file = ""
conf = ConfigParser({ # defaults
"mongo-db-name": "swift_usage",
"mongo-db-host": "localhost",
"mongo-db-port": "27017"
})
if os.path.exists("/etc/swift/swift-usage.conf"):
conf_file = "/etc/swift/swift-usage.conf"
conf.read(conf_file)
# load config from the config file if it exists
mongo_db_name = conf.get('DEFAULT', 'mongo-db-name')
mongo_db_host = conf.get('DEFAULT', 'mongo-db-host')
mongo_db_port = int(conf.get('DEFAULT', 'mongo-db-port'))
db = pymongo.Connection(
mongo_db_host,
mongo_db_port)[mongo_db_name] # use the swift_usage database
def index(request):
funding_exhibits = Exhibit.objects.funding(
) | Exhibit.objects.full_fund_pause()
try:
funding_exhibits = funding_exhibits.select_related('item')\
.filter(item__categories=ItemCategory.objects.order_by('sort')[0])\
.order_by('-amount_funded')
except IndexError:
funding_exhibits = None
bidding_exhibits = Exhibit.objects.bidding() \
| Exhibit.objects.after_win_pause() \
| Exhibit.objects.paused() \
| Exhibit.objects.auto_paused_last() \
| Exhibit.objects.relisted()
bidding_exhibits = bidding_exhibits.exclude(in_queue=True)\
.select_related('item')\
.select_related('last_bidder_member')
# we need to display giveaways at 4 and 5 bidding positions
bidding_exhibits_giveaways = list(
bidding_exhibits.filter(item__giveaway=True))
bidding_exhibits_non_giveaways = list(
bidding_exhibits.exclude(item__giveaway=True))
if len(bidding_exhibits_non_giveaways) > 3:
bidding_exhibits = bidding_exhibits_non_giveaways[:3] \
+ bidding_exhibits_giveaways \
+ bidding_exhibits_non_giveaways[3:]
else:
bidding_exhibits = bidding_exhibits_non_giveaways + bidding_exhibits_giveaways
win_limit_time_left = (
request.user.win_limit_time_left) if request.user.is_authenticated(
) and request.user.is_on_win_limit() else None
categories = ItemCategory.objects.order_by('sort')
# get last 15 chat messages from MongoDB
connection = pymongo.Connection(settings.MONGODB['HOST'],
settings.MONGODB['PORT'])
db = getattr(connection, settings.MONGODB['NAME'])
chat_messages = list(db.chat.find().sort("date",
pymongo.DESCENDING).limit(15))
chat_messages.reverse()
# chat state from mongo dynamic settings
guest_chat_state = db.settings.find_one({'name': 'GUEST_CHAT_STATE'})
guest_chat_state = guest_chat_state.get('status',
1) if guest_chat_state else 1
# temporary disable small bidding templates
# if bidding_exhibits.count() > settings.MAX_AUCTIONS_BIG_TEMPLATES_COUNT or request.mobile:
# bidding_exhibits_template = 'exhibit/bidding_box_small.html'
# bidding_exhibits_refund_template = 'exhibit/bidding_refund_box_small.html'
# mobile_version = 1
# else:
# bidding_exhibits_template = 'exhibit/bidding_box.html'
# bidding_exhibits_refund_template = 'exhibit/bidding_refund_box.html'
# mobile_version = 0
mobile_version = 0
bidding_exhibits_template = 'exhibit/bidding_box.html'
return render(
request,
'exhibit/index.html',
{
'funding_exhibits': funding_exhibits,
'bidding_exhibits': bidding_exhibits,
'categories': categories,
'chat_messages': chat_messages,
'win_limit_time_left': win_limit_time_left,
'bidding_exhibits_template': bidding_exhibits_template,
# 'bidding_exhibits_refund_template': bidding_exhibits_refund_template,
'mobile_version': mobile_version,
'guest_chat_state': guest_chat_state,
})
def __init__(self, address, name, entry_type):
self.address = format_address(address)
self.connection = mongo.Connection(*self.address)
self.data = self.connection[MONGO_DB_NAME][name]
self.entry_type = entry_type
def tushare_to_mongo():
conn = pymongo.Connection('127.0.0.1', port=27017)
df_index = ts.get_tick_data('600848', date='2014-12-22')
df = df_index.reset_index()
conn.db.tickdata.insert(json.loads(df.to_json(orient='records')))
import sys
import pymongo
import time
import subprocess
import multiprocessing
from datetime import datetime
cpu_count = multiprocessing.cpu_count()
# obtain a mongo connection
connection = pymongo.Connection('mongodb://localhost', safe=True)
# obtain a handle to the random database
db = connection.random
collection = db.randomData
total_documents_count = 50 * 1000 * 1000
inserted_documents_count = 0
sleep_seconds = 1
sleep_count = 0
for i in range(cpu_count):
documents_number = str(total_documents_count / cpu_count)
print documents_number
subprocess.Popen(
['python', '../create_random.py', documents_number,
str(i)])
start = datetime.now()
import pymongo
import time
import re
import string
import servers
import db
con = pymongo.Connection('001.shrd007.mongo.ktpd.xd.com', 27018)
names = con.database_names()
eachCnt = 100000
sleepInterval = 30
leftMon = 1
from bson.objectid import ObjectId
import datetime
#while 1:
if (True):
coList = con["replay_data"].collection_names()
for name in coList:
#if (name == "xd_s110.files"):
if (not (-1 == name.find("files"))):
print name
datas = name.split(".")
vname = datas[0]
tname = vname.replace("_new", "")
chunkName = vname + ".chunks"
leftReplay = []
r = servers.GetServerByName(tname)
if (0 == r):
continue
d = db.Database(r, False)
d.Use('demon_winners')
for item in d.Collection.find():
job_id = '1'
if len(sys.argv) < 2:
sys.exit("You must supply the item_number argument")
elif len(sys.argv) > 2:
job_id = sys.argv[2]
documents_number = int(sys.argv[1])
batch_number = 5 * 1000
job_name = 'Job#' + job_id
start = datetime.now()
# obtain a mongo connection
connection = pymongo.Connection("mongodb://localhost", safe=True)
# obtain a handle to the random database
db = connection.random
collection = db.randomData
batch_documents = [i for i in range(batch_number)]
for index in range(documents_number):
try:
date = datetime.fromtimestamp(
time.mktime(min_date.timetuple()) +
int(round(random.random() * delta)))
value = random.random()
document = {
'created_on': date,
# -*- coding: utf-8 -*-
import config
import color
import re
import pymongo, sys, os
from collections import Counter, defaultdict
from itertools import product
from pprint import pprint
import logging
print >> sys.stderr, '[info] init ...',
sys.stderr.flush()
db = pymongo.Connection(config.mongo_addr)[config.db_name]
# get all emotions
emotions = sorted(
[x['emotion'] for x in db['emotions'].find({'label': 'LJ40K'})])
eids = dict(
enumerate(
sorted([x['emotion']
for x in db['emotions'].find({'label': 'LJ40K'})])))
print >> sys.stderr, 'ok'
# setting_id = '537af6923681dff466c19e38'
# root = 'tmp'
intersection = False
def accuracy(res, ratio=1):
TP = res['TP']
TN = res['TN'] / float(ratio)
# coding=UTF-8
# !/usr/bin/env python
import datetime
import pymongo
import logging
import os
from copy import deepcopy
LOG_SERVER_NAME = 'timer01'
host = '10.27.228.73'
port = 30001
uname = 'PS_ztcjlAdmin'
pwd = 'PS_managerZtcjl'
web_pyconn = pymongo.Connection(host, port)
web_db = web_pyconn.jl_ztcjl
web_db.authenticate(uname, pwd)
tlog_coll = web_db.timer_log
base_path = os.path.dirname(os.path.realpath(__file__))
parent_path = os.path.split(base_path)[0]
conf_file = "logger.conf"
logging.config.fileConfig("%s/%s" % (parent_path, conf_file))
log = logging.getLogger()
log.setLevel(logging.INFO)
class AnalysisLog():
"""docstring for AnalysisLog"""
def ext_pillar(collection='pillar',
id_field='_id',
re_pattern=None,
re_replace='',
fields=None):
"""
Connect to a mongo database and read per-node pillar information.
Parameters:
* `collection`: The mongodb collection to read data from. Defaults to
``'pillar'``.
* `id_field`: The field in the collection that represents an individual
minon id. Defaults to ``'_id'``.
* `re_pattern`: If your naming convention in the collection is shorter
than the minion id, you can use this to trim the name.
`re_pattern` will be used to match the name, and `re_replace` will
be used to replace it. Backrefs are supported as they are in the
Python standard library. If ``None``, no mangling of the name will
be performed - the collection will be searched with the entire
minion id. Defaults to ``None``.
* `re_replace`: Use as the replacement value in node ids matched with
`re_pattern`. Defaults to ''. Feel free to use backreferences here.
* `fields`: The specific fields in the document to use for the pillar
data. If ``None``, will use the entire document. If using the
entire document, the ``_id`` field will be converted to string. Be
careful with other fields in the document as they must be string
serializable. Defaults to ``None``.
"""
conn = pymongo.Connection(__opts__['mongo.host'], __opts__['mongo.port'])
db = conn[__opts__['mongo.db']]
user = __opts__.get('mongo.user')
password = __opts__.get('mongo.password')
if user and password:
db.authenticate(user, password)
# Do the regex string replacement on the minion id
minion_id = __opts__['id']
if re_pattern:
minion_id = re.sub(re_pattern, re_replace, minion_id)
log.info("ext_pillar.mongo: looking up pillar def for {'%s': '%s'} "
"in mongo" % (id_field, minion_id))
pillar = db[collection].find_one({id_field: minion_id}, fields=fields)
if pillar:
if fields:
log.debug(
"ext_pillar.mongo: found document, returning fields '%s'" %
fields)
else:
log.debug("ext_pillar.mongo: found document, returning whole doc")
if '_id' in pillar:
# Converting _id to a string
# will avoid the most common serialization error cases, but DBRefs
# and whatnot will still cause problems.
pillar['_id'] = str(pillar['_id'])
return pillar
else:
# If we can't find the minion the database it's not necessarily an
# error.
log.debug("ext_pillar.mongo: no document found in collection %s" %
collection)
return {}
