def add_to_list(listid, listitem):
mongo = Connection(dbhost, dbport)
db = mongo.msl
db.auth(dbusr, dbpass)
lists = db.lists
lists.update({"listid":listid}, {"$push": {"text":listitem}}, upsert=True)
mongo.end_request()
def remove_from_list(listid, listitems):
mongo = Connection(dhost, dbport)
db = mongo.msl
db.aut(dbuser, dbpass)
lists = db.lists
lists.update({"listid":listid}, {"$pullAll": {"text":listitems}})
mongo.end_request()
class DB(object):
def __init__(self):
self.db_connection = Connection()
self.db = self.db_connection.hasadna.data
self.db.ensure_index([("m.path",1), ("m.id",1)])
def find_one(self,query_dict):
return self.db.find_one(query_dict)
def find(self,query_dict,fields=None,limit=None,start=None):
return self.db.find(query_dict,limit=limit,skip=start,fields=fields)
def count(self,*args,**kwargs):
return self.find(*args,**kwargs).count()
def save(self,doc):
pass
#self.db.save(doc,safe=True)
def remove(self,query_dict):
pass
#self.db.remove(query_dict,safe=True)
def after_request(self):
self.db_connection.end_request()
class DB(object):
def __init__(self):
self.db_connection = Connection()
self.db = self.db_connection.hasadna.data
self.db.ensure_index([("m.path", 1), ("m.id", 1)])
def find_one(self, query_dict):
return self.db.find_one(query_dict)
def find(self, query_dict, fields=None, limit=None, start=None):
return self.db.find(query_dict, limit=limit, skip=start, fields=fields)
def count(self, *args, **kwargs):
return self.find(*args, **kwargs).count()
def save(self, doc):
pass
#self.db.save(doc,safe=True)
def remove(self, query_dict):
pass
#self.db.remove(query_dict,safe=True)
def after_request(self):
self.db_connection.end_request()
def create():
mongo = Connection(dbhost, dbport)
db = mongo.msl
db.auth(dbuser, dbpass)
lists = db.lists
list = {}
listid = "".join(sample(string.lowercase + string.digits, 6))
list["listid"] = listid
mongo.end_request()
return redirect('/view/' + listid)
def view(listid):
if request.method == 'POST':
listitem = request.form['listitem']
add_to_list(listid, listitem)
mongo = Connection(dbhost, dbport)
db = mongo.msl
db.auth(dbuser, dbpass)
lists = db.lists
entry = lists.find_one({"listid": listid})
mongo.end_request()
entries = entry['text'] if entry else ''
return render_template('list.html', title=title, listid=listid, entries=entries)
class Database(object):
def __init__(self, uri, dbname):
self._connection = Connection(uri)
self._db = self._connection[dbname]
def collection(self, name):
return self._db[name]
def deref(self, ref):
return self._db.dereference(ref)
def free(self):
self._connection.end_request()
class MongoObject(object):
def __init__(self, app=None):
if app is not None:
self.app = app
self.init_app(app)
self.Model = self.make_model()
self.mapper = {}
def init_app(self, app):
app.config.setdefault('MONGODB_HOST', "mongodb://localhost:27017")
app.config.setdefault('MONGODB_DATABASE', "")
app.config.setdefault('MONGODB_AUTOREF', True)
# initialize connection and Model properties
self.app = app
self.connect()
self.app.after_request(self.close_connection)
def connect(self):
self.connection = Connection(self.app.config['MONGODB_HOST'])
def init_connection(self):
self.connection = Connection(self.app.config['MONGODB_HOST'])
def make_model(self):
model = Model
model.query = _QueryProperty(self)
return model
@property
def session(self):
if not getattr(self, "db", None):
self.db = self.connection[self.app.config['MONGODB_DATABASE']]
if self.app.config['MONGODB_AUTOREF']:
self.db.add_son_manipulator(NamespaceInjector())
self.db.add_son_manipulator(AutoReferenceObject(self))
return self.db
def set_mapper(self, model):
# Set up mapper for model, so when ew retrieve documents from database,
# we will know how to map them to model object based on `_ns` fields
self.mapper[model.__collection__] = model
def close_connection(self, response):
self.connection.end_request()
return response
def clear(self):
self.connection.drop_database(self.app.config['MONGODB_DATABASE'])
self.connection.end_request()
class RpcClient(object):
def __init__(self,host='localhost',port=27018):
self.__connection = Connection(host,port,max_pool_size=10)
return
def __getattr__(self,funcname):
if funcname.startswith('_'):
return "you are trying to call %s from our backend"%funcname
else:
func = lambda *args,**kwargs:self.__call__(funcname,*args,**kwargs)
return func
def __call__(self,funcname,*args,**kwargs):
querydict = {'funcname':funcname,'argstr':(args,kwargs)}
try:
collection = self.__connection['backend']['rpc']
ret = collection.find_one(querydict)
if ret.get('$error',None):
raise BackendError(ret['$error']['message'],ret['$error']['detail'])
return ret['$result'] or None
finally:
if self.__connection:
self.__connection.end_request()
def main(filein):
def cleanquit():
pbar.finish()
print "waiting for queue"
inqueue.join()
print str(block) + " blocks (" + str(block * blocksize) + " bytes) read in total"
print str(len(seenhashes)) + " unique blocks (" + str(len(seenhashes) * blocksize) + " bytes) seen"
print str(sum(mongoblocks)) + " blocks written to database\n"
sys.exit()
def get_file_size(filename):
"Get the file size by seeking at end"
fd= os.open(filename, os.O_RDONLY)
try:
return os.lseek(fd, 0, os.SEEK_END)
finally:
os.close(fd)
if chunksize % blocksize != 0:
print "Error: chunksize must be a multiple of blocksize"
sys.exit(1)
filesize = get_file_size(filein)
inqueue = Queue.Queue(10)
mongoconn = Connection(host = mongohost, port = 27017)
mongodb = mongoconn.circuitbackup
mongofiles = mongodb.files
mongofiles.ensure_index([("fileName", ASCENDING), ("uploadDate", DESCENDING)])
mongofiles.insert({"_id": files_id, "chunkSize": blocksize, "length": filesize, "uploadDate": datetime.datetime.utcnow(), "complete": False, "fileName": filein, "hostName": platform.node()})
start = time.time()
for i in xrange(5):
t = DiskReadThread(inqueue, mongoconn)
t.daemon = True
t.start()
widgets = ['Progress: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets, maxval=filesize).start()
fin = open(filein, 'rb')
block = 0
while True:
try:
inblock = fin.read(chunksize)
if not inblock:
break
else:
inblocksize = len(inblock)
if inblocksize == chunksize or inblocksize % blocksize == 0:
blocks = inblocksize / blocksize
elif inblocksize > blocksize:
blocks = inblocksize / blocksize + 1
else:
blocks = 1
inqueue.put((inblock, block, blocks))
#if block != 0 and block % (chunksize / blocksize * 10) == 0:
pbar.update(block * blocksize)
block += blocks
except KeyboardInterrupt:
print "KeyboardInterrupt"
cleanquit()
pbar.finish()
print "waiting for queue"
inqueue.join()
mongofiles.update({"_id": files_id}, { "$set" : { "complete" : True } })
mongoconn.end_request()
print str(block) + " blocks read from input"
print str(len(seenhashes)) + " unique blocks (" + str(len(seenhashes) * blocksize) + " bytes) seen"
print str(sum(mongoblocks)) + " blocks written to database"
print "seen hashes set consumed " + str(sys.getsizeof(seenhashes)) + " bytes"
end = time.time()
elapsed = end - start
print "elapsed time" + str(elapsed)
class MongoSet(object):
""" This class is used to control the MongoSet integration
to Flask application.
Adds :param db: and :param _fallback_lang: into Model
Usage:
app = Flask(__name__)
mongo = MongoSet(app)
This class also provides access to mongo Model:
class Product(mongo.Model):
structure = t.Dict({
'title': t.String,
'quantity': t.Int,
'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)),
}).allow_extra('*')
indexes = ['id']
via register method:
mongo = MongoSet(app)
mongo.register(Product, OtherModel)
or via decorator:
from flask.ext.mongoset import Model
@mongo.register
class Product(Model):
pass
"""
def __init__(self, app=None):
self.Model = Model
self.connection = None
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app):
self.app = app
self._configure(app)
self.connection = self._get_connection()
if not hasattr(self.app, 'extensions'):
self.app.extensions = {}
self.app.extensions['mongoset'] = _MongoSetState(self, self.app)
self.Model.db = self.session
self.Model._fallback_lang = app.config['MONGODB_FALLBACK_LANG']
# 0.9 and later
if hasattr(self.app, 'teardown_appcontext'):
teardown = self.app.teardown_appcontext
# 0.7 to 0.8
elif hasattr(self.app, 'teardown_request'):
teardown = self.app.teardown_request
# Older Flask versions
else:
teardown = self.app.after_request
@teardown
def close_connection(response):
state = get_state(self.app)
if state.connection is not None:
state.connection.end_request()
return response
def _configure(self, app):
for key, value in default_config.items():
app.config.setdefault('MONGODB_{}'.format(key), value)
def get_app(self, reference_app=None):
"""Helper method that implements the logic to look up an application.
"""
if reference_app is not None:
return reference_app
if self.app is not None:
return self.app
ctx = connection_stack.top
if ctx is not None:
return ctx.app
raise RuntimeError('application not registered on db '
'instance and no application bound '
'to current context')
def _get_connection(self):
"""Connect to the MongoDB server and register the documents from
:attr:`registered_documents`. If you set ``MONGODB_USERNAME`` and
``MONGODB_PASSWORD`` then you will be authenticated at the
``MONGODB_DATABASE``.
"""
app = self.get_app()
self.connection = Connection(
host=app.config['MONGODB_HOST'],
port=app.config['MONGODB_PORT'],
slave_okay=app.config['MONGODB_SLAVE_OKAY'])
return self.connection
def register(self, *models):
"""Register one or more :class:`mongoset.Model` instances to the
connection.
"""
for model in models:
if not model.db or not isinstance(model.db, Database):
setattr(model, 'db', self.session)
model.indexes and model.query.ensure_index(model.indexes)
return len(models) == 1 and models[0] or models
@property
def session(self):
""" Returns MongoDB
"""
app = self.get_app()
state = get_state(app)
db = state.connection[app.config['MONGODB_DATABASE']]
if app.config['MONGODB_USERNAME']:
auth_success = db.authenticate(
app.config['MONGODB_USERNAME'],
app.config['MONGODB_PASSWORD'])
if not auth_success:
raise AuthenticationError("can't connect to data base,"
" wrong user_name or password")
db.add_son_manipulator(NamespaceInjector())
db.add_son_manipulator(SavedObject())
if app.config['MONGODB_AUTOREF']:
db.add_son_manipulator(AutoReference(db))
return db
def clear(self):
self.connection.drop_database(self.app.config['MONGODB_DATABASE'])
self.connection.end_request()
class MongoSet(object):
""" This class is used to control the MongoObject integration
to Flask application.
Adds :param db: and :param _fallback_lang: into Model
Usage:
app = Flask(__name__)
mongo = MongoObject(app)
This class also provides access to mongo Model:
class Product(mongo.Model):
structure = t.Dict({
'title': t.String,
'quantity': t.Int,
'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)),
}).allow_extra('*')
indexes = ['id']
via register method:
mongo = MongoObject(app)
mongo.register(Product, OtherModel)
or via decorator:
from flaskext.mongoobject import Model
@mongo.register
class Product(Model):
pass
"""
def __init__(self, app=None):
self.Model = Model
if app is not None:
self.init_app(app)
def init_app(self, app):
app.config.setdefault('MONGODB_HOST', "localhost")
app.config.setdefault('MONGODB_PORT', 27017)
app.config.setdefault('MONGODB_DATABASE', "")
app.config.setdefault('MONGODB_AUTOREF', False)
app.config.setdefault('AUTOINCREMENT', True)
app.config.setdefault('FALLBACK_LANG', 'en')
self.app = app
self.connect()
self.app.after_request(self.close_connection)
self.Model.db = self.session
self.Model._fallback_lang = app.config.get('FALLBACK_LANG')
def connect(self):
"""Connect to the MongoDB server and register the documents from
:attr:`registered_documents`. If you set ``MONGODB_USERNAME`` and
``MONGODB_PASSWORD`` then you will be authenticated at the
``MONGODB_DATABASE``.
"""
if not getattr(self, 'app', None):
raise RuntimeError('The mongoobject extension was not init to '
'the current application. Please make sure '
'to call init_app() first.')
if not getattr(self, 'connection', None):
self.connection = Connection(
host=self.app.config.get('MONGODB_HOST'),
port=self.app.config.get('MONGODB_PORT'),
slave_okay=self.app.config.get('MONGODB_SLAVE_OKAY', False))
if self.app.config.get('MONGODB_USERNAME') is not None:
auth_success = self.session.authenticate(
self.app.config.get('MONGODB_USERNAME'),
self.app.config.get('MONGODB_PASSWORD'))
if not auth_success:
raise AuthenticationIncorrect("can't connect to data base, wrong user_name or password")
def register(self, *models):
"""Register one or more :class:`mongoobject.Model` instances to the
connection.
"""
for model in models:
if not getattr(model, 'db', None) or not isinstance(model.db, Database):
setattr(model, 'db', self.session)
setattr(model, '_fallback_lang', self.app.config.get('FALLBACK_LANG'))
model.indexes and model.query.ensure_index(model.indexes)
return len(models) == 1 and models[0] or models
@property
def session(self):
""" Returns MongoDB
"""
if not getattr(self, "db", None):
self.db = self.connection[self.app.config['MONGODB_DATABASE']]
self.db.add_son_manipulator(NamespaceInjector())
if self.app.config['MONGODB_AUTOREF']:
self.db.add_son_manipulator(AutoReferenceObject(self))
else:
self.db.add_son_manipulator(SavedObject())
if self.app.config['AUTOINCREMENT']:
self.db.add_son_manipulator(AutoincrementId())
return self.db
def close_connection(self, response):
self.connection.end_request()
return response
def clear(self):
self.connection.drop_database(self.app.config['MONGODB_DATABASE'])
self.connection.end_request()
class MongoSet(object):
""" This class is used to control the MongoSet integration
to Flask application.
Adds :param db: and :param _fallback_lang: into Model
Usage:
app = Flask(__name__)
mongo = MongoSet(app)
This class also provides access to mongo Model:
class Product(mongo.Model):
structure = t.Dict({
'title': t.String,
'quantity': t.Int,
'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)),
}).allow_extra('*')
indexes = ['id']
via register method:
mongo = MongoSet(app)
mongo.register(Product, OtherModel)
or via decorator:
from flask.ext.mongoset import Model
@mongo.register
class Product(Model):
pass
"""
def __init__(self, app=None):
self.Model = Model
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app):
app.config.setdefault('MONGODB_HOST', "localhost")
app.config.setdefault('MONGODB_PORT', 27017)
app.config.setdefault('MONGODB_USERNAME', '')
app.config.setdefault('MONGODB_PASSWORD', '')
app.config.setdefault('MONGODB_DATABASE', "")
app.config.setdefault('MONGODB_AUTOREF', False)
app.config.setdefault('MONGODB_AUTOINCREMENT', False)
app.config.setdefault('MONGODB_FALLBACK_LANG', 'en')
app.config.setdefault('MONGODB_SLAVE_OKAY', False)
self.app = app
if not hasattr(app, 'extensions'):
app.extensions = {}
app.extensions['mongoset'] = self
self.connect()
@app.teardown_appcontext
def close_connection(response):
state = get_state(app)
if state.connection is not None:
state.connection.end_request()
return response
self.Model.db = self.session
self.Model._fallback_lang = app.config.get('MONGODB_FALLBACK_LANG')
def connect(self):
"""Connect to the MongoDB server and register the documents from
:attr:`registered_documents`. If you set ``MONGODB_USERNAME`` and
``MONGODB_PASSWORD`` then you will be authenticated at the
``MONGODB_DATABASE``.
"""
if not hasattr(self, 'app'):
raise RuntimeError('The mongoset extension was not init to '
'the current application. Please make sure '
'to call init_app() first.')
if not hasattr(self, 'connection'):
self.connection = Connection(
host=self.app.config.get('MONGODB_HOST'),
port=self.app.config.get('MONGODB_PORT'),
slave_okay=self.app.config.get('MONGODB_SLAVE_OKAY', False))
if self.app.config.get('MONGODB_USERNAME') is not '':
auth_success = self.session.authenticate(
self.app.config.get('MONGODB_USERNAME'),
self.app.config.get('MONGODB_PASSWORD'))
if not auth_success:
raise AuthenticationError("can't connect to data base,"
" wrong user_name or password")
def register(self, *models):
"""Register one or more :class:`mongoset.Model` instances to the
connection.
"""
for model in models:
if not model.db or not isinstance(model.db, Database):
setattr(model, 'db', self.session)
model.indexes and model.query.ensure_index(model.indexes)
setattr(model, '_fallback_lang',
self.app.config['MONGODB_FALLBACK_LANG'])
return len(models) == 1 and models[0] or models
@property
def session(self):
""" Returns MongoDB
"""
if not hasattr(self, "db"):
self.db = self.connection[self.app.config['MONGODB_DATABASE']]
# we need namespaces in any case
self.db.add_son_manipulator(NamespaceInjector())
if self.app.config['MONGODB_AUTOREF']:
self.db.add_son_manipulator(AutoReference(self.db))
if self.app.config['MONGODB_AUTOINCREMENT']:
self.db.add_son_manipulator(AutoincrementId())
self.db.add_son_manipulator(SavedObject())
return self.db
def clear(self):
self.connection.drop_database(self.app.config['MONGODB_DATABASE'])
self.connection.end_request()