def get_db(db_name): import pymongo DB_HOST = ["localhost"] DB_PORT = 27017 db = pymongo.Connection(DB_HOST, DB_PORT)[db_name] return db
- Split existing texts - Rewrites existing links - Rewrites history """ import sys import os import re from pprint import pprint import pymongo p = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, p) sys.path.insert(0, p + "/sefaria") from sefaria.texts import * connection = pymongo.Connection() db = connection[SEFARIA_DB] db.authenticate(SEFARIA_DB_USER, SEFARIA_DB_PASSWORD) # Insert index records from JSON mishneh_torah = json.loads(open("mishneh-torah.json", "r").read()) for topic in mishneh_torah: db.index.save(topic) # Build the mapping of numbered sections to new text names index = [] indices = db.index.find({"categories": "Mishneh Torah"}).sort([["order.0", 1]]) section = -1
import pymongo # Setup Mongo conn = pymongo.Connection() # defaults to localhost db = conn.agile_data emails = db['emails'] email_list = emails.find()[0:20] for email in email_list: print email
def log_items_to_person_proposals(start, end, found_cb): single_map_cb = Code(""" function() { emit(this.i, {pid: [this.p]}); } """) single_reduce_cb = Code(""" function(item, vals) { var res = {pid:[]}; for(var v in vals) { var value = vals[v]; if(value.pid !== undefined) { for(var j in value.pid) { res.pid.push(value.pid[j]); } } } return res; } """) date_query = {'d': {"$gte": start, "$lt": end}, 'p': {'$exists': True}} db_host = config.get('download_log', 'server') db_port = config.getint('download_log', 'port') connection = pymongo.Connection(db_host, db_port) db = connection[config.get('download_log', 'db_name')] db_name_single_items = config.get('download_log', 'single_collection') db_name_cart_items = config.get('download_log', 'cart_collection') db_name_uniq_cart_items = config.get('download_log', 'uniq_cart_collection') db_name_uniq_items = config.get('download_log', 'uniq_single_collection') db_name_user2proposals = config.get('download_log', 'user2proposals_collection') cart_map_cb = Code(""" function() { emit(this.c, {pid: [this.p]}); } """) cart_reduce_cb = Code(""" function(item, vals) { var res = {pid:[]}; for(var v in vals) { var value = vals[v]; if(value.pid !== undefined) { for(var j in value.pid) { res.pid.push(value.pid[j]); } } } return res; } """) db[db_name_cart_items].ensure_index([('d', pymongo.ASCENDING)]) db[db_name_cart_items].map_reduce(cart_map_cb, cart_reduce_cb, out={'replace': db_name_uniq_cart_items}, query=date_query) def cart_process(chunk_items, collection): sql = "SELECT cart_id, item_id FROM myemsl.cart_items WHERE cart_id in (%s) ORDER by cart_id;" % ( ','.join([str(i) for i in chunk_items])) cnx = myemsldb_connect(myemsl_schema_versions=['1.3']) cursor = cnx.cursor() cursor.execute(sql) old_cart_id = None cart_list = {} for row in cursoriter(cursor): toset = cart_list.setdefault(row[0], []) toset.insert(0, row[1]) for key in cart_list.iterkeys(): collection.update({'_id': key}, {"$set": { 'value.i': cart_list[key] }}, w=1) chunk_items = [] collection = db[db_name_uniq_cart_items] for item in db[db_name_uniq_cart_items].find(fields={'_id': 1}): chunk_items.insert(0, int(item['_id'])) if len(chunk_items) >= chunk_size: cart_process(chunk_items, collection) chunk_items = [] if len(chunk_items) > 0: cart_process(chunk_items, collection) db[db_name_single_items].ensure_index([('d', pymongo.ASCENDING)]) db[db_name_single_items].map_reduce(single_map_cb, single_reduce_cb, out={'replace': db_name_uniq_items}, query=date_query) map2_cb = Code(""" function() { for(var i in this.value.i) { emit(this.value.i[i], {pid: this.value.pid}); } } """) reduce2_cb = Code(""" function(item, vals) { var res = {p:[], pid:[]}; for(var v in vals) { var value = vals[v]; if(value.p !== undefined) { for(var j in value.p) { res.p.push(value.p[j]); } } if(value.pid !== undefined) { for(var j in value.pid) { res.pid.push(value.pid[j]); } } } return res; } """) db[db_name_uniq_cart_items].map_reduce(map2_cb, reduce2_cb, out={'reduce': db_name_uniq_items}) def proposals_process(chunk_items, collection): query = { "fields": ["proposals"], "query": { "constant_score": { "filter": { "ids": { "values": chunk_items } } } } } server = config.get('elasticsearch', 'server') results = json.loads( call_curl("%s/myemsl_current_simple_items/simple_items/_search" % (server), method='POST', idata=json.dumps(query))) if results['hits']['total'] != len(chunk_items): raise Exception('Failed to find some items') for hit in results['hits']['hits']: if 'fields' in hit and 'proposals' in hit['fields']: collection.update( {'_id': int(hit['_id'])}, {"$set": { 'value.p': hit['fields']['proposals'] }}, w=1) chunk_items = [] collection = db[db_name_uniq_items] for item in db[db_name_uniq_items].find(fields={'_id': 1}): chunk_items.insert(0, int(item['_id'])) if len(chunk_items) >= chunk_size: proposals_process(chunk_items, collection) chunk_items = [] if len(chunk_items) > 0: proposals_process(chunk_items, collection) map3_cb = Code(""" function() { for(var i in this.value.pid) { var pid = this.value.pid[i]; emit(pid, {p: this.value.p}); } } """) reduce3_cb = Code(""" function(item, vals) { var tp = {} var res = {p:[]}; for(var v in vals) { var value = vals[v]; for(var j in value.p) { tp[value.p[j]] = 1; } } for(var p in tp) { res.p.push(p); } return res; } """) db[db_name_uniq_items].map_reduce(map3_cb, reduce3_cb, out={'replace': db_name_user2proposals}) for item in db[db_name_user2proposals].find(): if item['value']['p'] != None: found_cb(int(item['_id']), item['value']['p'])
def initVdsMongo(self): self.vdsMongo = pymongo.Connection("bang")['freeway']['vds']
''' This file includes code to: Analyze the text content of a tweet object to build a training set using frequency analysis. Naive Bayes classifier used for sentiment analysis. ''' import pymongo as mongo import geo import simplejson as json import time connection = mongo.Connection() db = connection.CompProb word_collection = db.words keep = ['no', 'ok', 'up'] exclude = set([ "2012 election", "presidential election", "presidential debate", "US president", "obama administration", "president of the united states", "republican", "ron paul", "mitt romney", "mitt", "romney", "democrat", "democrats", "democratic", "barack obama", "obama", "barack", "about", "after", "all", "also", "and", "another", "any", "are", "because", "been", "before", "being", "between", "both", "came", "come", "could", "each", "for", "from", "get", "got", "has", "had", "have", "her", "here", "him", "himself", "his", "how", "into", "like", "make", "many", "might", "more", "most", "much", "must", "now", "only", "other", "our", "out", "over", "said", "same", "see", "should", "since", "some", "still", "such", "take", "than", "that", "the", "their", "them", "then", "there", "these", "they", "this", "those", "through", "too", "under", "very", "was", "way", "well", "were", "what", "where", "which", "while", "who", "with", "would", "you", "your"
def run(server, db, coll, method='find', query=None, limit=1000, fields=None, sort=None, fill=None): # Create an empty response object. response = tangelo.empty_response() # Check the requested method. if method not in ['find', 'insert']: response['error'] = "Unsupported MongoDB operation '%s'" % (method) return bson.json_util.dumps(response) # Decode the query strings into Python objects. try: if query is not None: query = decode(query, 'query', response) if fields is not None: fields = decode(fields, 'fields', response) if sort is not None: sort = decode(sort, 'sort', response) if fill is not None: fill = decode(fill, 'fill', response) else: fill = True except ValueError: return bson.json_util.dumps(response) # Cast the limit value to an int. try: limit = int(limit) except ValueError: response[ 'error'] = "Argument 'limit' ('%s') could not be converted to int." % ( limit) return bson.json_util.dumps(response) # Create database connection. try: c = pymongo.Connection(server)[db][coll] except pymongo.errors.AutoReconnect: response['error'] = "Could not connect to MongoDB server '%s'" % ( server) return bson.json_util.dumps(response) # Perform the requested action. if method == 'find': # Do a find operation with the passed arguments. it = c.find(spec=query, fields=fields, limit=limit, sort=sort) # Create a list of the results. if fill: results = [x for x in it] else: results = [] # Create an object to structure the results. retobj = {} retobj['count'] = it.count() retobj['data'] = results # Pack the results into the response object, and return it. response['result'] = retobj else: raise RuntimeError("illegal method '%s' in module 'mongo'") # Return the response object. return bson.json_util.dumps(response)
import pymongo import random from pyokapi import * import threading import time _conn = pymongo.Connection(MONGO_HOST, MONGO_PORT) _apis = _conn.okapi.service _deploy = _conn.okapi.deploys _docs = _conn.okapi.doc dkrt = "http://172.17.0.1:2375" def get_all_containers(): return InvokeService( dkrt + '/containers/json?all=1&filters={"status":["running"]}').get() def create_container(author, runtime): name = "%s.%s" % (author, runtime) if runtime == "python": image = "ls/okapi-py2" elif runtime == "python3": image = "ls/okapi-py3" elif runtime == "java": image = "ls/okapi-java" else: raise Exception("not supported runtime: %s" % runtime) d = { "Hostname": name,
def prepare_db(self): self.connection = pymongo.Connection(self.args.host, self.args.port) self.db = self.connection[self.args.database]
#!/bin/python #run file like this #execfile('/xchip/cogs/hogstrom/scripts/dose_timeCourse/ts_query.py') #cd /xchip/cogs/hogstrom/scripts/dose_timeCourse import pymongo conn = pymongo.Connection('vitalstatistix') affogato = conn['affogato'] affogato.authenticate('cmap_user', 'l1000') signature = affogato['signature'] ### QUERY 1 ### #signature.find({brew_prefix:{$in:[/^DOS/,/^PAC/]}},{sig_id:1}).count() qCurs = signature.find_one({'det_plate':'DOS059_PC3_24H_X2_F3B4_DUO52HI53LO','is_gold':1.0},{'cell_id':True,'pert_type':True,'sig_id':True,'pert_desc':True,'is_gold':True,'pert_dose':True,'pert_time':True,'cell_id':True}) qLis = [] for x in qCurs: qLis.append(x) ### QUERY 2 ### testGene = 'NOTCH2' qCurs = signature.find({'pert_type':'trt_sh','pert_desc':testGene,'is_gold':1.0},{'cell_id':True,'pert_type':True,'sig_id':True,'pert_desc':True,'is_gold':True,'pert_dose':True,'pert_time':True,'cell_id':True}) qLis = [] for x in qCurs: qLis.append(x) sigs = [str(x['sig_id']) for x in qLis] isGlds = [str(x['is_gold']) for x in qLis] dose = [str(x['pert_dose']) for x in qLis]
def add_mongo_db(config): #for use mongodb: db_url = urlparse(config.registry.settings['mongo_uri']) conn = pymongo.Connection(host=db_url.hostname) config.registry.settings['db_conn'] = conn
import pymongo import time import math db = pymongo.Connection('localhost',27017).bbarnes_test earth = db.universe.find_one({'name' : "Earth"}) x = earth['x_pos'] y = earth['y_pos'] r = earth['distance_from_star'] name = earth['name'] id = earth['_id'] parent_id = earth['parent_id'] parent = db.universe.find_one({ '_id' : parent_id}) star_x = parent['x_pos'] star_y = parent['y_pos'] print "x is ",x print "y is ",y print "radius is ",r print "planet in question is " ,name print "its starting at " , x,y def x_increase():
#-*- coding:utf-8 -*- import sys import urllib2 import urllib import cookielib import hashlib import Image import pymongo from scrapy import Selector reload(sys) sys.setdefaultencoding('utf-8') # initialize mongodb connection try: con = pymongo.Connection('127.0.0.1', 27017) kd_db = con.kd table = kd_db.info except Exception as e: print 'Cannot connect mongodb!' print e exit(-1) def get_html_by_data(url, use_cookie=False): data = {} post_data = urllib.urlencode(data) cj = cookielib.CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) req = urllib2.Request(url) if use_cookie:
# -*- coding: utf-8 -*- import pymongo import os import sys import string import codecs import time ISOTIMEFORMAT='%Y-%m-%dT%XZ' #import math #connect c = "昌平镇" conn = pymongo.Connection("10.214.0.147",27017) #set database db = conn.Air #db1 = conn.Beijing #db.authenticate("pm","ccntgrid") collection = db.Stations #collection_w = db.Weather #collection_b = db1.BJ_Weather_Forecast #initialize the tmp result result = {} #4 of weather prediction #select result print "Ready" #ofile = codecs.open('X_1', 'w',"utf-8") ofile1 = codecs.open('station_Changping.txt', 'w',"utf-8") distinct_time = collection.distinct('time_point')
if len(sys.argv) < 4: usage_message = """usage: %s db coll f1 [f2] [f3...] For one or more files containing edx tracking logs, insert into the collection given. Files ending .gz they are decompressed on the fly. Files successfully loaded are tracked in coll_incremental. If already loaded, skip. """ sys.stderr.write(usage_message % sys.argv[0]) sys.exit(1) db_name = sys.argv[1] collection_name = sys.argv[2] # Get database connection and collections connection = pymongo.Connection('localhost', 27017) db = connection[db_name] events_coll = db[collection_name] imported_coll = db[collection_name + "_imported"] total_error = 0 total_success = 0 # Connect to course_structure database STRUCTURE_COLLECTION_NAME = 'course_structure' struct_coll = db[STRUCTURE_COLLECTION_NAME] # Append course_structure info to record dict def append_course_struct(id): try:
try: logging.info("Connecting to source MongoDB server: ".format( self.opts.src)) self.mongo_src = pymongo.Connection(host=self.opts.src) logging.info( "Connected successfully to source MongoDB server, version: {0}" .format(self.mongo_src.server_info()['version'])) except pymongo.errors.ConnectionFailure, e: logging.error( "Could not connect to source MongoDB server - {0}".format(e)) sys.exit(0) try: logging.info("Connecting to destination MongoDB server: ".format( self.opts.dst)) self.mongo_dst = pymongo.Connection(host=self.opts.dst) logging.info( "Connected successfully to destination MongoDB server, version: {0}" .format(self.mongo_dst.server_info()['version'])) except pymongo.errors.ConnectionFailure, e: logging.error( "Could not connect to destination MongoDB server - {0}".format( e)) sys.exit(0) def start_sharding(self): logging.debug("START SHARDING") self.connect_to_servers() self.setup_database_shards() logging.debug("END SHARDING")
import json from collections import defaultdict import codecs import pymongo client = pymongo.Connection() db = client.xuetangx category_stats = db.category_stats category2courses = defaultdict(list) course_info = json.load(open('course_info.json')) course2image = json.load(open('course2image.json')) for course_id, course in course_info.items(): if course['category'] is None: continue categories = [x[1] for x in json.loads(course['category']).items()] for category in categories: category2courses[category].append({ 'course_id': course_id, 'name': course['name'], 'image': course2image.get(course_id, None) }) category_db = list() for category, courses in category2courses.items(): category_db.append({'category': category, 'courses': courses})
# coding: utf-8 import json import os import pymongo from warnings import warn config = {} try: try: config_file = os.environ["POPCORN_CONF"] except KeyError: config_file = "popcorn.conf" with open(config_file) as fh: config = json.load(fh) del config_file except (IOError, ValueError): warn("I need configuration!") db = pymongo.Connection()["popcorn"] try: _db_config = {"url": "mongodb://127.0.0.1", "name": "popcorn"} _db_config.update(config.get("database", {})) db = pymongo.Connection(_db_config["url"])[_db_config["name"]] del _db_config except pymongo.errors.AutoReconnect: warn("I need database!") db.messages.ensure_index([("channel", 1), ("time", -1)])
data = "".join(str(x) for x in data if str(x).isalpha()) if (timestamp - lastTime) / 1000 > 10: lastTime = timestamp lat = 40.48 + (random.random() / 5 - .2) lon = -74.440 - (random.random() / 5 - .2) return [{ "timestamp": timestamp, "user": data, "lat": lat, "lon": lon }, lastTime] print "Bam,nice swipe " + data + "!" return [-1, lastTime] dbConnection = pymongo.Connection("172.31.74.128", 27017) #default port, change if ne errors = ["block", "thenticat", "response", "AAA", "FF", "ff"] db = dbConnection.db1 brate = 115200 ser = serial.Serial('/dev/ttyACM0', brate) while (1): temp = ser.readline() a = [x for x in errors if temp.find(x) != -1] if len(a) == 0: temp, lastTime = parseData(temp, lastTime) ser.flushInput() if temp != -1: print temp print "Bam,into the db" db.dataSet.save(temp)
def filterFunction(jobid,filename,inputPtnPath,model,table,partAns,st,domainRange): contentPtnJson = json.load(open(os.path.join(inputPtnPath,filename),"r")) print "Worker %d : Read %s into filter" % (jobid,filename) connect = pymongo.Connection() db = connect.projizz collection = db.result.yago.answer queries = map(lambda x: x[:-4], contentPtnJson) itr = collection.find({"revid":{"$in":queries}}) print "worker %d query=%d, result=%d" % (jobid,len(queries),itr.count()) count = 0 for ans in itr: count += 1 key = "%s.txt" % (ans["revid"]) types = ans["type"] # Now only consider properties, no references. relation = ans["properties"] ptnEx = contentPtnJson[key] relaEx = [] for line in ptnEx: for ptn in line[1]: ptnId = "%d" % (ptn[0]) rfp = table[ptnId]["relations"] # never seen pattern if not ptnId in st: continue # if only one relation if len(rfp) < 2: if st[ptnId][0][1]["support"] > 0 and not rfp[0] in relaEx: relaEx.append(rfp[0]) # more than one relation else: # using the first as the answer if st[ptnId][0][1]["support"] > 0 and not rfp[0] in relaEx: relaEx.append(rfp[0]) # Remove impossible relations toBeRemove = [] for attribute in relaEx: # speical case, produced if domainRange[attribute] == "": continue if not domainRange[attribute]["domain"] in types: if not attribute in toBeRemove: toBeRemove.append(attribute) for attribute in toBeRemove: relaEx.remove(attribute) # Evaluation for attribute in partAns: postive = False true = False if attribute in relaEx: postive = True if attribute in relation: true = True if true: if postive: partAns[attribute]["tp"].append(ans["revid"]) else: partAns[attribute]["fn"].append(ans["revid"]) else: if postive: partAns[attribute]["fp"].append(ans["revid"]) else: partAns[attribute]["tn"].append(ans["revid"]) if count % 100 == 0: print "worker #%d done %d." % (jobid,count) return partAns
import pymongo # setup: sudo pip install pymongo from ConfigParser import ConfigParser import os.path import sys conf_file = "" conf = ConfigParser({ # defaults "mongo-db-name": "swift_usage", "mongo-db-host": "localhost", "mongo-db-port": "27017" }) if os.path.exists("/etc/swift/swift-usage.conf"): conf_file = "/etc/swift/swift-usage.conf" conf.read(conf_file) # load config from the config file if it exists mongo_db_name = conf.get('DEFAULT', 'mongo-db-name') mongo_db_host = conf.get('DEFAULT', 'mongo-db-host') mongo_db_port = int(conf.get('DEFAULT', 'mongo-db-port')) db = pymongo.Connection( mongo_db_host, mongo_db_port)[mongo_db_name] # use the swift_usage database
def index(request): funding_exhibits = Exhibit.objects.funding( ) | Exhibit.objects.full_fund_pause() try: funding_exhibits = funding_exhibits.select_related('item')\ .filter(item__categories=ItemCategory.objects.order_by('sort')[0])\ .order_by('-amount_funded') except IndexError: funding_exhibits = None bidding_exhibits = Exhibit.objects.bidding() \ | Exhibit.objects.after_win_pause() \ | Exhibit.objects.paused() \ | Exhibit.objects.auto_paused_last() \ | Exhibit.objects.relisted() bidding_exhibits = bidding_exhibits.exclude(in_queue=True)\ .select_related('item')\ .select_related('last_bidder_member') # we need to display giveaways at 4 and 5 bidding positions bidding_exhibits_giveaways = list( bidding_exhibits.filter(item__giveaway=True)) bidding_exhibits_non_giveaways = list( bidding_exhibits.exclude(item__giveaway=True)) if len(bidding_exhibits_non_giveaways) > 3: bidding_exhibits = bidding_exhibits_non_giveaways[:3] \ + bidding_exhibits_giveaways \ + bidding_exhibits_non_giveaways[3:] else: bidding_exhibits = bidding_exhibits_non_giveaways + bidding_exhibits_giveaways win_limit_time_left = ( request.user.win_limit_time_left) if request.user.is_authenticated( ) and request.user.is_on_win_limit() else None categories = ItemCategory.objects.order_by('sort') # get last 15 chat messages from MongoDB connection = pymongo.Connection(settings.MONGODB['HOST'], settings.MONGODB['PORT']) db = getattr(connection, settings.MONGODB['NAME']) chat_messages = list(db.chat.find().sort("date", pymongo.DESCENDING).limit(15)) chat_messages.reverse() # chat state from mongo dynamic settings guest_chat_state = db.settings.find_one({'name': 'GUEST_CHAT_STATE'}) guest_chat_state = guest_chat_state.get('status', 1) if guest_chat_state else 1 # temporary disable small bidding templates # if bidding_exhibits.count() > settings.MAX_AUCTIONS_BIG_TEMPLATES_COUNT or request.mobile: # bidding_exhibits_template = 'exhibit/bidding_box_small.html' # bidding_exhibits_refund_template = 'exhibit/bidding_refund_box_small.html' # mobile_version = 1 # else: # bidding_exhibits_template = 'exhibit/bidding_box.html' # bidding_exhibits_refund_template = 'exhibit/bidding_refund_box.html' # mobile_version = 0 mobile_version = 0 bidding_exhibits_template = 'exhibit/bidding_box.html' return render( request, 'exhibit/index.html', { 'funding_exhibits': funding_exhibits, 'bidding_exhibits': bidding_exhibits, 'categories': categories, 'chat_messages': chat_messages, 'win_limit_time_left': win_limit_time_left, 'bidding_exhibits_template': bidding_exhibits_template, # 'bidding_exhibits_refund_template': bidding_exhibits_refund_template, 'mobile_version': mobile_version, 'guest_chat_state': guest_chat_state, })
def __init__(self, address, name, entry_type): self.address = format_address(address) self.connection = mongo.Connection(*self.address) self.data = self.connection[MONGO_DB_NAME][name] self.entry_type = entry_type
def tushare_to_mongo(): conn = pymongo.Connection('127.0.0.1', port=27017) df_index = ts.get_tick_data('600848', date='2014-12-22') df = df_index.reset_index() conn.db.tickdata.insert(json.loads(df.to_json(orient='records')))
import sys import pymongo import time import subprocess import multiprocessing from datetime import datetime cpu_count = multiprocessing.cpu_count() # obtain a mongo connection connection = pymongo.Connection('mongodb://localhost', safe=True) # obtain a handle to the random database db = connection.random collection = db.randomData total_documents_count = 50 * 1000 * 1000 inserted_documents_count = 0 sleep_seconds = 1 sleep_count = 0 for i in range(cpu_count): documents_number = str(total_documents_count / cpu_count) print documents_number subprocess.Popen( ['python', '../create_random.py', documents_number, str(i)]) start = datetime.now()
import pymongo import time import re import string import servers import db con = pymongo.Connection('001.shrd007.mongo.ktpd.xd.com', 27018) names = con.database_names() eachCnt = 100000 sleepInterval = 30 leftMon = 1 from bson.objectid import ObjectId import datetime #while 1: if (True): coList = con["replay_data"].collection_names() for name in coList: #if (name == "xd_s110.files"): if (not (-1 == name.find("files"))): print name datas = name.split(".") vname = datas[0] tname = vname.replace("_new", "") chunkName = vname + ".chunks" leftReplay = [] r = servers.GetServerByName(tname) if (0 == r): continue d = db.Database(r, False) d.Use('demon_winners') for item in d.Collection.find():
job_id = '1' if len(sys.argv) < 2: sys.exit("You must supply the item_number argument") elif len(sys.argv) > 2: job_id = sys.argv[2] documents_number = int(sys.argv[1]) batch_number = 5 * 1000 job_name = 'Job#' + job_id start = datetime.now() # obtain a mongo connection connection = pymongo.Connection("mongodb://localhost", safe=True) # obtain a handle to the random database db = connection.random collection = db.randomData batch_documents = [i for i in range(batch_number)] for index in range(documents_number): try: date = datetime.fromtimestamp( time.mktime(min_date.timetuple()) + int(round(random.random() * delta))) value = random.random() document = { 'created_on': date,
# -*- coding: utf-8 -*- import config import color import re import pymongo, sys, os from collections import Counter, defaultdict from itertools import product from pprint import pprint import logging print >> sys.stderr, '[info] init ...', sys.stderr.flush() db = pymongo.Connection(config.mongo_addr)[config.db_name] # get all emotions emotions = sorted( [x['emotion'] for x in db['emotions'].find({'label': 'LJ40K'})]) eids = dict( enumerate( sorted([x['emotion'] for x in db['emotions'].find({'label': 'LJ40K'})]))) print >> sys.stderr, 'ok' # setting_id = '537af6923681dff466c19e38' # root = 'tmp' intersection = False def accuracy(res, ratio=1): TP = res['TP'] TN = res['TN'] / float(ratio)
# coding=UTF-8 # !/usr/bin/env python import datetime import pymongo import logging import os from copy import deepcopy LOG_SERVER_NAME = 'timer01' host = '10.27.228.73' port = 30001 uname = 'PS_ztcjlAdmin' pwd = 'PS_managerZtcjl' web_pyconn = pymongo.Connection(host, port) web_db = web_pyconn.jl_ztcjl web_db.authenticate(uname, pwd) tlog_coll = web_db.timer_log base_path = os.path.dirname(os.path.realpath(__file__)) parent_path = os.path.split(base_path)[0] conf_file = "logger.conf" logging.config.fileConfig("%s/%s" % (parent_path, conf_file)) log = logging.getLogger() log.setLevel(logging.INFO) class AnalysisLog(): """docstring for AnalysisLog"""
def ext_pillar(collection='pillar', id_field='_id', re_pattern=None, re_replace='', fields=None): """ Connect to a mongo database and read per-node pillar information. Parameters: * `collection`: The mongodb collection to read data from. Defaults to ``'pillar'``. * `id_field`: The field in the collection that represents an individual minon id. Defaults to ``'_id'``. * `re_pattern`: If your naming convention in the collection is shorter than the minion id, you can use this to trim the name. `re_pattern` will be used to match the name, and `re_replace` will be used to replace it. Backrefs are supported as they are in the Python standard library. If ``None``, no mangling of the name will be performed - the collection will be searched with the entire minion id. Defaults to ``None``. * `re_replace`: Use as the replacement value in node ids matched with `re_pattern`. Defaults to ''. Feel free to use backreferences here. * `fields`: The specific fields in the document to use for the pillar data. If ``None``, will use the entire document. If using the entire document, the ``_id`` field will be converted to string. Be careful with other fields in the document as they must be string serializable. Defaults to ``None``. """ conn = pymongo.Connection(__opts__['mongo.host'], __opts__['mongo.port']) db = conn[__opts__['mongo.db']] user = __opts__.get('mongo.user') password = __opts__.get('mongo.password') if user and password: db.authenticate(user, password) # Do the regex string replacement on the minion id minion_id = __opts__['id'] if re_pattern: minion_id = re.sub(re_pattern, re_replace, minion_id) log.info("ext_pillar.mongo: looking up pillar def for {'%s': '%s'} " "in mongo" % (id_field, minion_id)) pillar = db[collection].find_one({id_field: minion_id}, fields=fields) if pillar: if fields: log.debug( "ext_pillar.mongo: found document, returning fields '%s'" % fields) else: log.debug("ext_pillar.mongo: found document, returning whole doc") if '_id' in pillar: # Converting _id to a string # will avoid the most common serialization error cases, but DBRefs # and whatnot will still cause problems. pillar['_id'] = str(pillar['_id']) return pillar else: # If we can't find the minion the database it's not necessarily an # error. log.debug("ext_pillar.mongo: no document found in collection %s" % collection) return {}