Ejemplo n.º 1
0
def add_to_list(listid, listitem):
 mongo = Connection(dbhost, dbport)
 db = mongo.msl
 db.auth(dbusr, dbpass)
 lists = db.lists
 lists.update({"listid":listid}, {"$push": {"text":listitem}}, upsert=True) 
 mongo.end_request()
Ejemplo n.º 2
0
def remove_from_list(listid, listitems):
 mongo = Connection(dhost, dbport)
 db = mongo.msl
 db.aut(dbuser, dbpass)
 lists = db.lists
 lists.update({"listid":listid}, {"$pullAll": {"text":listitems}}) 
 mongo.end_request()
Ejemplo n.º 3
0
class DB(object):
    
    def __init__(self):
        self.db_connection = Connection()
        self.db = self.db_connection.hasadna.data
        self.db.ensure_index([("m.path",1), ("m.id",1)])

    def find_one(self,query_dict):
        return self.db.find_one(query_dict)

    def find(self,query_dict,fields=None,limit=None,start=None):
        return self.db.find(query_dict,limit=limit,skip=start,fields=fields)

    def count(self,*args,**kwargs):
        return self.find(*args,**kwargs).count()

    def save(self,doc):
	pass
        #self.db.save(doc,safe=True)
        
    def remove(self,query_dict):
	pass
        #self.db.remove(query_dict,safe=True)
            
    def after_request(self):
        self.db_connection.end_request()
Ejemplo n.º 4
0
class DB(object):
    def __init__(self):
        self.db_connection = Connection()
        self.db = self.db_connection.hasadna.data
        self.db.ensure_index([("m.path", 1), ("m.id", 1)])

    def find_one(self, query_dict):
        return self.db.find_one(query_dict)

    def find(self, query_dict, fields=None, limit=None, start=None):
        return self.db.find(query_dict, limit=limit, skip=start, fields=fields)

    def count(self, *args, **kwargs):
        return self.find(*args, **kwargs).count()

    def save(self, doc):
        pass

    #self.db.save(doc,safe=True)

    def remove(self, query_dict):
        pass

    #self.db.remove(query_dict,safe=True)

    def after_request(self):
        self.db_connection.end_request()
Ejemplo n.º 5
0
def create():
 mongo = Connection(dbhost, dbport)
 db = mongo.msl
 db.auth(dbuser, dbpass)
 lists = db.lists
 list = {}
 listid = "".join(sample(string.lowercase + string.digits, 6))
 list["listid"] = listid
 mongo.end_request()
 return redirect('/view/' + listid)
Ejemplo n.º 6
0
def view(listid):
 if request.method == 'POST':
  listitem = request.form['listitem']
  add_to_list(listid, listitem)
 mongo = Connection(dbhost, dbport)
 db = mongo.msl
 db.auth(dbuser, dbpass)
 lists = db.lists
 entry = lists.find_one({"listid": listid})
 mongo.end_request()
 entries = entry['text'] if entry else ''
 return render_template('list.html', title=title, listid=listid, entries=entries)
Ejemplo n.º 7
0
class Database(object):

    def __init__(self, uri, dbname):
        self._connection = Connection(uri)
        self._db = self._connection[dbname]

    def collection(self, name):
        return self._db[name]

    def deref(self, ref):
        return self._db.dereference(ref)

    def free(self):
        self._connection.end_request()
Ejemplo n.º 8
0
class MongoObject(object):
    def __init__(self, app=None):
        if app is not None:
            self.app = app
            self.init_app(app)
        self.Model = self.make_model()
        self.mapper = {}

    def init_app(self, app):
        app.config.setdefault('MONGODB_HOST', "mongodb://localhost:27017")
        app.config.setdefault('MONGODB_DATABASE', "")
        app.config.setdefault('MONGODB_AUTOREF', True)
        # initialize connection and Model properties
        self.app = app
        self.connect()
        self.app.after_request(self.close_connection)

    def connect(self):
        self.connection = Connection(self.app.config['MONGODB_HOST'])

    def init_connection(self):
        self.connection = Connection(self.app.config['MONGODB_HOST'])

    def make_model(self):
        model = Model
        model.query = _QueryProperty(self)
        return model

    @property
    def session(self):
        if not getattr(self, "db", None):
            self.db = self.connection[self.app.config['MONGODB_DATABASE']]
            if self.app.config['MONGODB_AUTOREF']:
                self.db.add_son_manipulator(NamespaceInjector())
                self.db.add_son_manipulator(AutoReferenceObject(self))
        return self.db

    def set_mapper(self, model):
        # Set up mapper for model, so when ew retrieve documents from database,
        # we will know how to map them to model object based on `_ns` fields
        self.mapper[model.__collection__] = model

    def close_connection(self, response):
        self.connection.end_request()
        return response

    def clear(self):
        self.connection.drop_database(self.app.config['MONGODB_DATABASE'])
        self.connection.end_request()
Ejemplo n.º 9
0
class RpcClient(object):

    def __init__(self,host='localhost',port=27018):
        self.__connection = Connection(host,port,max_pool_size=10)
        return

    def __getattr__(self,funcname):
        if funcname.startswith('_'):
            return "you are trying to call %s from our backend"%funcname
        else:
            func = lambda *args,**kwargs:self.__call__(funcname,*args,**kwargs)
            return func

    def __call__(self,funcname,*args,**kwargs):
        querydict = {'funcname':funcname,'argstr':(args,kwargs)}
        try:
            collection = self.__connection['backend']['rpc']
            ret = collection.find_one(querydict)
            if ret.get('$error',None):
                raise BackendError(ret['$error']['message'],ret['$error']['detail'])
            return ret['$result'] or None
        finally:
            if self.__connection:
                self.__connection.end_request()
Ejemplo n.º 10
0
def main(filein):


    def cleanquit():

        pbar.finish()

        print "waiting for queue"
        inqueue.join()

        print str(block) + " blocks (" + str(block * blocksize) + " bytes) read in total"
        print str(len(seenhashes)) + " unique blocks (" + str(len(seenhashes) * blocksize) + " bytes) seen"
        print str(sum(mongoblocks)) + " blocks written to database\n"
        sys.exit()

    def get_file_size(filename):
        "Get the file size by seeking at end"
        fd= os.open(filename, os.O_RDONLY)
        try:
            return os.lseek(fd, 0, os.SEEK_END)
        finally:
            os.close(fd)



    if chunksize % blocksize != 0:
        print "Error: chunksize must be a multiple of blocksize"
        sys.exit(1)

    filesize = get_file_size(filein)

    inqueue = Queue.Queue(10)

    mongoconn = Connection(host = mongohost, port = 27017)

    mongodb = mongoconn.circuitbackup

    mongofiles = mongodb.files

    mongofiles.ensure_index([("fileName", ASCENDING), ("uploadDate", DESCENDING)])

    mongofiles.insert({"_id": files_id, "chunkSize": blocksize, "length": filesize, "uploadDate": datetime.datetime.utcnow(), "complete": False, "fileName": filein, "hostName": platform.node()})


    start = time.time()

    for i in xrange(5):

        t = DiskReadThread(inqueue, mongoconn)
        t.daemon = True
        t.start()

    widgets = ['Progress: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
               ' ', ETA(), ' ', FileTransferSpeed()]

    pbar = ProgressBar(widgets=widgets, maxval=filesize).start()

    fin = open(filein, 'rb')

    block = 0

    while True:

        try:

            inblock = fin.read(chunksize)
            if not inblock:
                break
            else:

                inblocksize = len(inblock)

                if inblocksize == chunksize or inblocksize % blocksize == 0:
                    blocks = inblocksize / blocksize
                elif inblocksize > blocksize:
                    blocks = inblocksize / blocksize + 1
                else:
                    blocks = 1

                inqueue.put((inblock, block, blocks))

                #if block != 0 and block % (chunksize / blocksize * 10) == 0:

                pbar.update(block * blocksize)

                block += blocks

        except KeyboardInterrupt:
            print "KeyboardInterrupt"
            cleanquit()




    pbar.finish()
    print "waiting for queue"
    inqueue.join()

    mongofiles.update({"_id": files_id}, { "$set" : { "complete" : True } })

    mongoconn.end_request()


    print str(block) + " blocks read from input"

    print str(len(seenhashes)) + " unique blocks (" + str(len(seenhashes) * blocksize) + " bytes) seen"

    print str(sum(mongoblocks)) + " blocks written to database"

    print "seen hashes set consumed " + str(sys.getsizeof(seenhashes)) + " bytes"

    end = time.time()

    elapsed = end - start

    print "elapsed time" + str(elapsed)
Ejemplo n.º 11
0
class MongoSet(object):
    """ This class is used to control the MongoSet integration
        to Flask application.
        Adds :param db: and :param _fallback_lang: into Model

    Usage:

        app = Flask(__name__)
        mongo = MongoSet(app)

    This class also provides access to mongo Model:

        class Product(mongo.Model):
            structure = t.Dict({
            'title': t.String,
            'quantity': t.Int,
            'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)),
        }).allow_extra('*')
        indexes = ['id']

    via register method:
        mongo = MongoSet(app)
        mongo.register(Product, OtherModel)

    or via decorator:
        from flask.ext.mongoset import Model

        @mongo.register
        class Product(Model):
            pass
    """
    def __init__(self, app=None):
        self.Model = Model
        self.connection = None

        if app is not None:
            self.init_app(app)
        else:
            self.app = None

    def init_app(self, app):
        self.app = app
        self._configure(app)

        self.connection = self._get_connection()

        if not hasattr(self.app, 'extensions'):
            self.app.extensions = {}
        self.app.extensions['mongoset'] = _MongoSetState(self, self.app)

        self.Model.db = self.session
        self.Model._fallback_lang = app.config['MONGODB_FALLBACK_LANG']

        # 0.9 and later
        if hasattr(self.app, 'teardown_appcontext'):
            teardown = self.app.teardown_appcontext
        # 0.7 to 0.8
        elif hasattr(self.app, 'teardown_request'):
            teardown = self.app.teardown_request
        # Older Flask versions
        else:
            teardown = self.app.after_request

        @teardown
        def close_connection(response):
            state = get_state(self.app)
            if state.connection is not None:
                state.connection.end_request()
            return response

    def _configure(self, app):
        for key, value in default_config.items():
            app.config.setdefault('MONGODB_{}'.format(key), value)

    def get_app(self, reference_app=None):
        """Helper method that implements the logic to look up an application.
        """
        if reference_app is not None:
            return reference_app
        if self.app is not None:
            return self.app
        ctx = connection_stack.top
        if ctx is not None:
            return ctx.app
        raise RuntimeError('application not registered on db '
                           'instance and no application bound '
                           'to current context')

    def _get_connection(self):
        """Connect to the MongoDB server and register the documents from
        :attr:`registered_documents`. If you set ``MONGODB_USERNAME`` and
        ``MONGODB_PASSWORD`` then you will be authenticated at the
        ``MONGODB_DATABASE``.
        """
        app = self.get_app()

        self.connection = Connection(
            host=app.config['MONGODB_HOST'],
            port=app.config['MONGODB_PORT'],
            slave_okay=app.config['MONGODB_SLAVE_OKAY'])

        return self.connection

    def register(self, *models):
        """Register one or more :class:`mongoset.Model` instances to the
        connection.
        """
        for model in models:
            if not model.db or not isinstance(model.db, Database):
                setattr(model, 'db', self.session)

            model.indexes and model.query.ensure_index(model.indexes)

        return len(models) == 1 and models[0] or models

    @property
    def session(self):
        """ Returns MongoDB
        """
        app = self.get_app()
        state = get_state(app)
        db = state.connection[app.config['MONGODB_DATABASE']]

        if app.config['MONGODB_USERNAME']:
            auth_success = db.authenticate(
                app.config['MONGODB_USERNAME'],
                app.config['MONGODB_PASSWORD'])
            if not auth_success:
                raise AuthenticationError("can't connect to data base,"
                                          " wrong user_name or password")

        db.add_son_manipulator(NamespaceInjector())
        db.add_son_manipulator(SavedObject())

        if app.config['MONGODB_AUTOREF']:
            db.add_son_manipulator(AutoReference(db))

        return db

    def clear(self):
        self.connection.drop_database(self.app.config['MONGODB_DATABASE'])
        self.connection.end_request()
Ejemplo n.º 12
0
class MongoSet(object):
    """ This class is used to control the MongoObject integration
        to Flask application.
        Adds :param db: and :param _fallback_lang: into Model

    Usage:

        app = Flask(__name__)
        mongo = MongoObject(app)

    This class also provides access to mongo Model:

        class Product(mongo.Model):
            structure = t.Dict({
            'title': t.String,
            'quantity': t.Int,
            'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)),
        }).allow_extra('*')
        indexes = ['id']

    via register method:
        mongo = MongoObject(app)
        mongo.register(Product, OtherModel)

    or via decorator:
        from flaskext.mongoobject import Model

        @mongo.register
        class Product(Model):
            pass
    """
    def __init__(self, app=None):
        self.Model = Model
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        app.config.setdefault('MONGODB_HOST', "localhost")
        app.config.setdefault('MONGODB_PORT', 27017)
        app.config.setdefault('MONGODB_DATABASE', "")
        app.config.setdefault('MONGODB_AUTOREF', False)
        app.config.setdefault('AUTOINCREMENT', True)
        app.config.setdefault('FALLBACK_LANG', 'en')
        self.app = app
        self.connect()
        self.app.after_request(self.close_connection)
        self.Model.db = self.session
        self.Model._fallback_lang = app.config.get('FALLBACK_LANG')

    def connect(self):
        """Connect to the MongoDB server and register the documents from
        :attr:`registered_documents`. If you set ``MONGODB_USERNAME`` and
        ``MONGODB_PASSWORD`` then you will be authenticated at the
        ``MONGODB_DATABASE``.
        """
        if not getattr(self, 'app', None):
            raise RuntimeError('The mongoobject extension was not init to '
                               'the current application.  Please make sure '
                               'to call init_app() first.')
        if not getattr(self, 'connection', None):
            self.connection = Connection(
                host=self.app.config.get('MONGODB_HOST'),
                port=self.app.config.get('MONGODB_PORT'),
                slave_okay=self.app.config.get('MONGODB_SLAVE_OKAY', False))

        if self.app.config.get('MONGODB_USERNAME') is not None:
            auth_success = self.session.authenticate(
                self.app.config.get('MONGODB_USERNAME'),
                self.app.config.get('MONGODB_PASSWORD'))
            if not auth_success:
                raise AuthenticationIncorrect("can't connect to data base, wrong user_name or password")

    def register(self, *models):
        """Register one or more :class:`mongoobject.Model` instances to the
        connection.
        """

        for model in models:
            if not getattr(model, 'db', None) or not isinstance(model.db, Database):
                setattr(model, 'db', self.session)
            setattr(model, '_fallback_lang', self.app.config.get('FALLBACK_LANG'))
            model.indexes and model.query.ensure_index(model.indexes)
        return len(models) == 1 and models[0] or models

    @property
    def session(self):
        """ Returns MongoDB
        """
        if not getattr(self, "db", None):
            self.db = self.connection[self.app.config['MONGODB_DATABASE']]
            self.db.add_son_manipulator(NamespaceInjector())
            if self.app.config['MONGODB_AUTOREF']:
                self.db.add_son_manipulator(AutoReferenceObject(self))
            else:
                self.db.add_son_manipulator(SavedObject())
            if self.app.config['AUTOINCREMENT']:
                self.db.add_son_manipulator(AutoincrementId())
        return self.db

    def close_connection(self, response):
        self.connection.end_request()
        return response

    def clear(self):
        self.connection.drop_database(self.app.config['MONGODB_DATABASE'])
        self.connection.end_request()
Ejemplo n.º 13
0
class MongoSet(object):
    """ This class is used to control the MongoSet integration
        to Flask application.
        Adds :param db: and :param _fallback_lang: into Model

    Usage:

        app = Flask(__name__)
        mongo = MongoSet(app)

    This class also provides access to mongo Model:

        class Product(mongo.Model):
            structure = t.Dict({
            'title': t.String,
            'quantity': t.Int,
            'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)),
        }).allow_extra('*')
        indexes = ['id']

    via register method:
        mongo = MongoSet(app)
        mongo.register(Product, OtherModel)

    or via decorator:
        from flask.ext.mongoset import Model

        @mongo.register
        class Product(Model):
            pass
    """
    def __init__(self, app=None):
        self.Model = Model

        if app is not None:
            self.init_app(app)
        else:
            self.app = None

    def init_app(self, app):
        app.config.setdefault('MONGODB_HOST', "localhost")
        app.config.setdefault('MONGODB_PORT', 27017)
        app.config.setdefault('MONGODB_USERNAME', '')
        app.config.setdefault('MONGODB_PASSWORD', '')
        app.config.setdefault('MONGODB_DATABASE', "")
        app.config.setdefault('MONGODB_AUTOREF', False)
        app.config.setdefault('MONGODB_AUTOINCREMENT', False)
        app.config.setdefault('MONGODB_FALLBACK_LANG', 'en')
        app.config.setdefault('MONGODB_SLAVE_OKAY', False)
        self.app = app
        if not hasattr(app, 'extensions'):
            app.extensions = {}
        app.extensions['mongoset'] = self
        self.connect()

        @app.teardown_appcontext
        def close_connection(response):
            state = get_state(app)
            if state.connection is not None:
                state.connection.end_request()
            return response

        self.Model.db = self.session
        self.Model._fallback_lang = app.config.get('MONGODB_FALLBACK_LANG')

    def connect(self):
        """Connect to the MongoDB server and register the documents from
        :attr:`registered_documents`. If you set ``MONGODB_USERNAME`` and
        ``MONGODB_PASSWORD`` then you will be authenticated at the
        ``MONGODB_DATABASE``.
        """
        if not hasattr(self, 'app'):
            raise RuntimeError('The mongoset extension was not init to '
                               'the current application.  Please make sure '
                               'to call init_app() first.')
        if not hasattr(self, 'connection'):
            self.connection = Connection(
                host=self.app.config.get('MONGODB_HOST'),
                port=self.app.config.get('MONGODB_PORT'),
                slave_okay=self.app.config.get('MONGODB_SLAVE_OKAY', False))

        if self.app.config.get('MONGODB_USERNAME') is not '':
            auth_success = self.session.authenticate(
                self.app.config.get('MONGODB_USERNAME'),
                self.app.config.get('MONGODB_PASSWORD'))
            if not auth_success:
                raise AuthenticationError("can't connect to data base,"
                                          " wrong user_name or password")

    def register(self, *models):
        """Register one or more :class:`mongoset.Model` instances to the
        connection.
        """
        for model in models:
            if not model.db or not isinstance(model.db, Database):
                setattr(model, 'db', self.session)

            model.indexes and model.query.ensure_index(model.indexes)

            setattr(model, '_fallback_lang',
                    self.app.config['MONGODB_FALLBACK_LANG'])

        return len(models) == 1 and models[0] or models

    @property
    def session(self):
        """ Returns MongoDB
        """
        if not hasattr(self, "db"):
            self.db = self.connection[self.app.config['MONGODB_DATABASE']]
            # we need namespaces in any case
            self.db.add_son_manipulator(NamespaceInjector())

            if self.app.config['MONGODB_AUTOREF']:
                self.db.add_son_manipulator(AutoReference(self.db))

            if self.app.config['MONGODB_AUTOINCREMENT']:
                self.db.add_son_manipulator(AutoincrementId())

            self.db.add_son_manipulator(SavedObject())
        return self.db

    def clear(self):
        self.connection.drop_database(self.app.config['MONGODB_DATABASE'])
        self.connection.end_request()