from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.contrib.auth.decorators import login_required
import datetime
from pymongo.connection import Connection
db = Connection().browner
@login_required
def main(request):
browns = db.browns.find_one({'browned': request.user.username},
{'browns': 1})
numbers = ["one", "two", "three", "four"]
tasklist = []
if browns:
for k in range(0, len(numbers)):
try:
tasklist.append({
"number": numbers[k],
"brown": browns['browns'][k]['brown']
})
except IndexError:
break
return render_to_response('browner.html', {'tasklist': tasklist},
context_instance=RequestContext(request))
currentdir = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, currentdir + "/../..")
from config_parser import config_parser
scp = config_parser('#', '=')
params = scp.parse_config('plugin.cfg')
print params
mongo_host = params['mongo_host']
mongo_port = int(params['mongo_port'])
logs_limit = int(params['logs_limit'])
logs_type = [item.strip() for item in params['logs_type'].split(',')]
logs_hosts = [item.strip() for item in params['logs_hosts'].split(',')]
logs_services = [item.strip() for item in params['logs_services'].split(',')]
con = Connection(mongo_host, mongo_port)
db = con['logs']
app = None # app is not otherwise
if not db:
message = "Error : Unable to connect to mongo database"
print message
# return {'app': app, 'eue_data': {}, 'message': message }
# 1/ count records
print "Logs are : %d" % db.logs.count()
print "Host notifications are : %d" % db.logs.find({
'type': 'HOST NOTIFICATION'
}).count()
today = datetime.datetime.now()
today_beginning = datetime.datetime(today.year, today.month, today.day, 0, 0,
#!/usr/loca/bin/python
from pymongo.connection import Connection
from pymongo.database import Database
from pymongo.collection import Collection
import FreebaseApi
import AlchemyApiCategory
import nerd
import DbPediaApi
# constants and globals
MONGO_CON = '107.22.242.25:27017'
mongo_con = Connection(MONGO_CON)
MONGO_DB = 'queries'
mongo_db = Database(mongo_con, MONGO_DB)
MONGO_COL = 'hott_rends'
mongo_col = Collection(mongo_db, MONGO_COL)
# for specific data
LOG_FILE = '/home/penser/tmp/specifics.txt'
# for simple data
STAT_PAPER = '/home/penser/tmp/stats.txt'
# number of queries
LIMIT = 420
# threshold to cut off unusual types
THRESHOLD = 2
class HotTrendsAnalysisAl:
''' docs '''
def __init__(self):
def setUp(self):
conn = Connection(pair)
response = conn.admin.command('ismaster')
if 'setName' in response:
raise SkipTest()
| 1 | 102 | 3 |
| 1 | 103 | 2 |
| 2 | 101 | 2 |
| 2 | 102 | 2.5 |
| 2 | 103 | 5 |
| 2 | 104 | 2 |
| 3 | 101 | 2.5 |
| 3 | 104 | 4 |
| 3 | 105 | 4.5 |
| 3 | 107 | 5 |
| 4 | 101 | 5 |
| 4 | 103 | 3 |
| 4 | 104 | 4.5 |
| 4 | 106 | 4 |
| 5 | 101 | 4 |
| 5 | 102 | 3 |
| 5 | 103 | 2 |
| 5 | 104 | 4 |
| 5 | 105 | 3.5 |
| 5 | 106 | 0 |
"""
db = Connection('127.0.0.1', 27017)['right-channel']
db['demo_ratings'].remove()
for line in data1.splitlines():
fields = [field.strip() for field in line.split(',') if field.strip()]
u = fields[0]
m = fields[1]
r = float(fields[2])
db['demo_ratings'].insert({'user_id': u, 'movie_id': m, 'rating': r})
'propagate': False,
},
'django.db.backends': {
'handlers': ['default-db'],
'level': 'DEBUG',
'propagate': False,
},
},
}
#IMPORT LOCAL SETTINGS
#=====================
try:
from settings_local import *
except:
pass
#CONNECT MONGODB
#===============
#Connect on MongoDB Database
from pymongo.connection import Connection
from pymongo.errors import ConnectionFailure
import sys
try:
connection = Connection(MG_HOST, MG_PORT)
DBCON = connection[MG_DB_NAME]
except ConnectionFailure, e:
sys.stderr.write("Could not connect to MongoDB: %s" % e)
sys.exit(1)
def run(self):
global generator
data = " "
connection = Connection(parser.get('mongodb', 'server'))
db = None
exec('db = connection.' + parser.get('mongodb', 'db'))
while self.__running__ and data:
data = str(self.__conn__.recv(1024 * 32)).decode("utf-8",
errors='ignore')
self.status()
if data == "NEXT":
ticker = generator.next()
self.__last__ = ticker
self.__conn__.send(str("PAGEID %i" % (ticker)).encode())
elif "FAIL" in data:
i = int(re.sub("FAIL ", '', data))
generator.logfail(self.__last__)
elif "ERROR" in data:
self._print("%-100s" % (" %30s %s" % (self.__client__, data)))
elif "FATAL" in data:
self._print("%-100s" % (" %30s NODE DOWN '%s'" %
(self.__client__, data)))
self.join()
elif "OKAY" in data:
if (self.__fail_count__ < self.__fail_limit__):
try:
i = int(data.split(' ', 1)[1])
self.__conn__.send('GOAHEAD'.encode())
except Exception as e:
exit(1)
card = None
try:
card = pickle.loads(self.__conn__.recv(i))
self.__fail_count__ = 0
self.__conn__.send('1'.encode())
self.__count__ += 1
except Exception as e:
self.__fail_count__ += 1
self.__conn__.send('0'.encode())
if card is not None:
card['_id'] = str(self.__last__)
db.cards.insert(card)
self._print("%-100s" % (" %30s DOWNLOADED CARD %i : %s" % \
(self.__client__, self.__last__, card['name'])))
else:
self.__conn__.send('1'.encode())
self.__fail_count__ = 0
self.__failures__ += 1
self._print("%-100s" % (" %30s FAILES TO DOWNLOADED CARD %i" % \
(self.__client__, self.__last__)))
if (not data):
self._print("%-100s" % (" %30s NO TRAFFIC, EXITING" %
(self.__client__)))
self.join()
def test_getters(self):
self.assertEqual(Connection(self.host, self.port).host, self.host)
self.assertEqual(Connection(self.host, self.port).port, self.port)
self.assertEqual(set([(self.host, self.port)]),
Connection(self.host, self.port).nodes)
import sys
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
import datetime
import objects
import os
from pymongo.connection import Connection
import logging
import humongolus as orm
import humongolus.widget as widget
from humongolus.field import FieldException
conn = Connection()
FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=FORMAT, level=logging.DEBUG)
logger = logging.getLogger("humongolus")
orm.settings(logger=logger, db_connection=conn)
class Field(unittest.TestCase):
def setUp(self):
self.name = "Anne"
self.genitalia = "inny"
self.obj = objects.Female()
self.job = objects.Job()
self.loca = objects.LocationGeo()
self.location = objects.Location()
def test_host_w_port(self):
self.assert_(Connection("%s:%d" % (self.host, self.port)))
self.assertRaises(ConnectionFailure, Connection,
"%s:1234567" % (self.host, ), self.port)
def test_repr(self):
self.assertEqual(repr(Connection(self.host, self.port)),
"Connection('%s', %s)" % (self.host, self.port))
def test_timeouts(self):
conn = Connection(self.host, self.port, connectTimeoutMS=300)
self.assertEquals(0.3, conn._Connection__pool.conn_timeout)
conn = Connection(self.host, self.port, socketTimeoutMS=300)
self.assertEquals(0.3, conn._Connection__pool.net_timeout)
def get_connection(*args, **kwargs):
host = os.environ.get("DB_IP", "localhost")
port = int(os.environ.get("DB_PORT", 27017))
return Connection(host, port, *args, **kwargs)
def _connect_mongodb(settings: Settings, mongodb: MongoDB) -> None:
if Connection is None:
raise Exception(
"Could not initialize MongoDB (Python-Modules are missing)")
mongodb.connection = Connection(*_mongodb_local_connection_opts(settings))
mongodb.db = mongodb.connection.__getitem__(os.environ["OMD_SITE"])
def mongo_upgrade():
"""Migrate mongodb schema and data."""
# Read reporting.conf to fetch mongo configuration.
config = Config(cfg=os.path.join("..", "..", "conf", "reporting.conf"))
# Run migration only if mongo is enabled as reporting module.
if config.mongodb.enabled:
host = config.mongodb.get("host", "127.0.0.1")
port = config.mongodb.get("port", 27017)
database = config.mongodb.get("db", "cuckoo")
user = config.mongodb.get("user", None)
password = config.mongodb.get("password", None)
print("Mongo reporting is enabled, strarting mongo data migration.")
if not port.isnumber():
print("Port must be an integer")
sys.exit()
# Support old Mongo.
try:
from pymongo.connection import Connection
from pymongo.errors import ConnectionFailure
conn = Connection(host, port)
db = conn.cuckoo
done = True
except ImportError:
print(
"Unable to import pymongo (install with `pip3 install pymongo`)"
)
done = False
except ConnectionFailure:
print("Cannot connect to MongoDB")
sys.exit()
try:
if not done:
import pymongo
try:
db = pymongo.MongoClient(host,
port=port,
username=user,
password=password,
authSource=database)[database]
except pymongo.errors.ConnectionFailure:
print("Cannot connect to MongoDB")
sys.exit()
except ImportError:
print(
"Unable to import pymongo (install with `pip3 install pymongo`)"
)
sys.exit()
# Check for schema version and create it.
if "cuckoo_schema" in db.collection_names():
print("Mongo schema version not expected")
sys.exit()
else:
db.cuckoo_schema.save({"version": mongo_revision})
else:
print("Mongo reporting module not enabled, skipping mongo migration.")
#!/usr/bin/env python2
from pymongo.connection import Connection
import socket
import threading
import pickle
import string
import time
import re
global db
connection = Connection("146.6.213.39")
db = connection.magic
class connectionThread(threading.Thread):
def __init__(self, conn):
threading.Thread.__init__(self)
self.__conn__ = conn
self.__running__ = True
self.__last__ = 0
self.__client__ = socket.gethostbyaddr(
self.__conn__.getpeername()[0])[0]
def join(self):
self.__running__ = False
self.__conn__.send("DONE".encode())
#print("[ %30s ] Thread killed..." % (self.__client__))
try:
threading.Thread.join(self)
except:
def init_messaging_agents():
services = {
'gcm': {},
'wns': {},
'apns': {},
'mpns': {},
'sms': {},
}
mongodb = None
while not mongodb:
try:
mongodb = Connection(options.mongohost, options.mongoport)
except Exception as ex:
logging.error(ex)
masterdb = mongodb[options.masterdb]
apps = masterdb.applications.find()
for app in apps:
''' APNs setup '''
services['apns'][app['shortname']] = []
conns = int(app['connections'])
if conns < 1:
conns = 1
if 'environment' not in app:
app['environment'] = 'sandbox'
if file_exists(app.get('certfile', False)) and file_exists(app.get('keyfile', False)) and 'shortname' in app:
if app.get('enableapns', False):
for instanceid in range(0, conns):
try:
apn = APNClient(app['environment'], app['certfile'], app['keyfile'], app['shortname'], instanceid)
except Exception as ex:
logging.error(ex)
continue
services['apns'][app['shortname']].append(apn)
''' GCMClient setup '''
services['gcm'][app['shortname']] = []
if 'gcmprojectnumber' in app and 'gcmapikey' in app and 'shortname' in app:
try:
http = GCMClient(app['gcmprojectnumber'], app['gcmapikey'], app['shortname'], 0)
except Exception as ex:
logging.error(ex)
continue
services['gcm'][app['shortname']].append(http)
''' WNS setup '''
services['wns'][app['shortname']] = []
if 'wnsclientid' in app and 'wnsclientsecret' in app and 'shortname' in app:
try:
wns = WNSClient(masterdb, app, 0)
except Exception as ex:
logging.error(ex)
continue
services['wns'][app['shortname']].append(wns)
''' MPNS setup '''
services['mpns'][app['shortname']] = []
try:
mpns = MPNSClient(masterdb, app, 0)
except Exception as ex:
logging.error(ex)
continue
services['mpns'][app['shortname']].append(mpns)
''' clickatell '''
services['sms'][app['shortname']] = []
try:
sms = ClickatellClient(masterdb, app, 0)
except Exception as ex:
logging.error(ex)
continue
services['sms'][app['shortname']].append(sms)
mongodb.close()
return services
def _connect(self):
settings = copy.deepcopy(self.settings_dict)
def pop(name, default=None):
return settings.pop(name) or default
db_name = pop('NAME')
host = pop('HOST')
port = pop('PORT')
user = pop('USER')
password = pop('PASSWORD')
options = pop('OPTIONS', {})
self.operation_flags = options.pop('OPERATIONS', {})
if not any(k in ['save', 'delete', 'update']
for k in self.operation_flags):
# Flags apply to all operations.
flags = self.operation_flags
self.operation_flags = {'save': flags, 'delete': flags,
'update': flags}
# Lower-case all OPTIONS keys.
for key in options.iterkeys():
options[key.lower()] = options.pop(key)
attempts = 0
while True:
try:
try:
self.connection = Connection(host=host, port=port, **options)
self.database = self.connection[db_name]
except TypeError:
exc_info = sys.exc_info()
raise ImproperlyConfigured, exc_info[1], exc_info[2]
if user and password:
if not self.database.authenticate(user, password):
raise ImproperlyConfigured("Invalid username or password.")
self.connected = True
connection_created.send(sender=self.__class__, connection=self)
''' execute a quick sample query so that the failure will happen
on the command that is run after the switchover, auth succeeds
on secondary but commands cannot be run. This command will
throw an exception and hence we will attempt to reconnect again '''
self.database['system.indexes'].find_one()
break
except Exception as e:
print 'MongoConnectionFailure to database %s %s' % (db_name,str(e))
print traceback.format_exc()
''' Make sure we set connected to False just in case we failed on the send '''
self.connected = False
''' initialize these instance variables, so that we can delete them '''
''' this will ensure that the __get_attr__ class method properly reconnects '''
self.database = None
self.connection = None
del self.database
del self.connection
attempts += 1
if attempts < self.conn_retries:
time.sleep(self.conn_sleep_interval)
print 'MongoConnectionRetry attempt=%d' % attempts
continue
raise e
# -*- coding: utf-8 -*-
import sys
import json
import os
import gzip
from pymongo.connection import Connection
hgm = Connection(port=37010).hgm.families
saving = True
count = 0
def do_import(F):
degree, weight, A, B, hodge, gal2, gal3, gal5, gal7 = F
A.sort()
B.sort()
Astr = '.'.join([str(x) for x in A])
Bstr = '.'.join([str(x) for x in B])
label = "A%s_B%s" % (Astr, Bstr)
data = {
'label': label,
'degree': degree,
'weight': weight,
'A': A,
'B': B,
'hodge': hodge,
'gal2': gal2,
'gal3': gal3,
'gal5': gal5,
def setUp(self):
conn = Connection(pair)
response = conn.admin.command('ismaster')
if 'setName' in response:
raise SkipTest("Connected to a replica set, not a standalone mongod")
# -*- coding: utf-8 -*-
import sys
import time
import bson
import sage.all
from sage.all import *
from pymongo.connection import Connection
fields = Connection(port=dbport).numberfields.fields
fields = Connection(port=dbport).numberfields.fields
saving = True
def coeffs(s):
return [a for a in s[1:-1].split(',')]
def base_label(d, r1, D, ind):
return str(d) + "." + str(r1) + "." + str(abs(D)) + "." + str(ind)
def makeb(n, t):
return bson.SON([('n', n), ('t', t)])
def makes(n, t):
return '%02d,%03d' % (n, t)
def makels(li):
def __init__(self,db,collection):
self.conn = Connection()
self.db = self.conn[db]
self.collection = self.db[collection]
'''
Created on Jan 13, 2013
@author: Fang Jiaguo
'''
from pyes.es import ES
from pymongo.connection import Connection
import json
settings = json.load(open('settings.json', 'r'))
mongodb = Connection(settings['mongodb']['host'],
settings['mongodb']['port'])[settings['mongodb']['db']]
elasticsearch = ES(('http', settings['elasticsearch']['host'],
settings['elasticsearch']['port']))
#!/usr/bin/env python3
# -*- coding: utf-8 *-*
from pymongo.connection import Connection
from configparser import SafeConfigParser
if __name__ == '__main__':
global parser
parser = SafeConfigParser()
parser.read('settings.ini')
connection = Connection(parser.get('mongodb', 'server'))
db = None
exec("db = connection." + parser.get('mongodb', 'db'))
print("generating name set...")
names = set(n['name'] for n in db.cards.find({}, {'name':1}))
print("done!")
print(type(names))
print(names)
for name in names:
try:
high = db.cards.find({"name": name},
{"muid":1}).sort("muid",-1).limit(1)[0]['muid']
dups = db.cards.find({'name': name, 'muid': {'$lt': high}}).count()
db.cards.remove({'name': name, 'muid': {'$lt': high}})
if(dups):
ret = [[parseSingle(i) for i in row] for row in ret]
if len(ret) == 1:
ret = ret[0]
elif all([len(row) == 1 for row in ret]):
ret = [row[0] for row in ret]
return parsed_as, ret
if len(sys.argv) == 2:
file1 = open(sys.argv[1], "r")
print guessParsing(file1)[1]
quit()
db = Connection(port=dbport)
db = Connection(port=dbport)
fs = GridFS(db.upload)
for entry in db.upload.fs.files.find(
{
"$or": [{
"metadata.status": "approved"
}, {
"metadata.status": "approvedchild"
}]
},
sort=[("uploadDate", -1)]):
print '%s: %s (%s)' % (entry['_id'], entry['filename'],
str(entry['uploadDate']))
name = entry['metadata']['uploader_id'] + str(entry['_id'])
file = fs.get(entry['_id'])
where
- coeffs: (list of ints) a polredabs'ed polynomial defining the field, so
[3,2,1] represents x^2+2*x+3
- clgp: (list of ints) the class group structure [a_1,a_2,...] where
"""
import sys, time
import re
import json
import sage.all
from sage.all import os
from pymongo.connection import Connection
forms = Connection(port=37010).halfintegralmf.forms
saving = True
def sd(f):
for k in f.keys():
print '%s ---> %s' % (k, f[k])
def makels(li):
li2 = [str(x) for x in li]
return ','.join(li2)
def string2list(s):
driver.DeleteDataSource(output_shape)
# connection information
mongodb_server = 'localhost'
mongodb_port = 27017
mongodb_db = 'gisdb'
mongodb_collection = '50m'
# 1. first we import features from shapefile to mongodb
print 'Importing data to mongodb...'
shape2mongodb(input_shape, mongodb_server, mongodb_port, mongodb_db,
mongodb_collection, True, 'STATE in (40,41,42)')
# 2. then we export features from mongodb to shapefile
print 'Exporting data from mongodb...'
mongodb2shape(mongodb_server, mongodb_port, mongodb_db, mongodb_collection,
output_shape, {"STATE": "40"})
# 3. now some test with mongodb
connection = Connection(mongodb_server, mongodb_port)
print 'Getting database MongoDB %s' % mongodb_db
db = connection[mongodb_db]
print 'Getting the collection %s' % mongodb_collection
collection = db[mongodb_collection]
# counting the collection
print 'Elements in collection: %s' % collection.count()
# finding one feature
feature = collection.find_one()
print 'Here is one random feature that has been stored:'
print feature
# some query now
print 'There are %s counties in STATE = 40' % collection.find({
"STATE": "40"
}).count()
def open_db(host, db):
conn = Connection(host)
try:
yield conn[db]
finally:
conn.disconnect()
def __init__(self,
db_name='test',
host='127.0.0.1',
port=27017,
gen_ids=True,
quiet_output=False,
safe_mode=False,
schemes=()):
# set quiet output if need
Selector.quiet_output = quiet_output
# set read-only attrs
self._name = db_name
self._host = host
self._port = port
# set attrs
self.gen_ids = gen_ids
self.quiet_output = quiet_output
self.safe_mode = bool(safe_mode)
self.safe_opts = safe_mode \
if safe_mode and isinstance(safe_mode, dict) else {}
# connect to mongo
self._connection = Connection(host, port)[db_name]
# add converter
self._connection.add_son_manipulator(ConvertToObject(self))
# declare vars
self._collections = {}
collections = set()
# take the collections that belong to this database
# or do not belong to anything
for scheme in schemes:
if len(scheme._meta.fields) and \
getattr(scheme._meta, 'db', self.name) == self.name:
collections.add(scheme)
# create plural maker
plural = Plural()
# set aliases and references for each collection
for coll in collections:
# occupy collection and set reference to this database
setattr(coll._meta, 'db', self._name)
setattr(coll._meta, 'db_ref', self)
if getattr(coll._meta, 'alias', '').startswith('_'):
raise SchemaError.metadata_naming_exc(coll._meta.alias,
coll.__module__,
coll.__name__)
if not hasattr(coll._meta, 'alias'):
# no alias was given
coll._meta.alias = plural.make(coll.__name__)
if not hasattr(self, coll.__name__):
# collection name is free
setattr(self, coll.__name__, coll)
elif hasattr(self, coll._meta.alias):
# collection name or alias must be free
raise SchemaError.collection_binding_exc(coll)
if not hasattr(self, coll._meta.alias):
# alias name is free
setattr(self, coll._meta.alias, coll)
# add to database collections by provider name
self._collections[coll.provider] = coll
from beaker.middleware import CacheMiddleware, SessionMiddleware
from beaker.exceptions import InvalidCacheBackendError
from nose import SkipTest
from webtest import TestApp
from pymongo.connection import Connection
import unittest
import time
try:
clsmap['mongodb_gridfs']._init_dependencies()
except InvalidCacheBackendError:
raise SkipTest("an appropriate mmongodb backend is not installed")
uri = 'mongodb://localhost/test.beaker'
_mongo = Connection('localhost').test
def setup():
_mongo.beaker.chunks.drop()
_mongo.beaker.files.drop()
def teardown():
import shutil
shutil.rmtree('./cache', True)
_mongo.beaker.chunks.drop()
_mongo.beaker.files.drop()
def simple_session_app(environ, start_response):
