示例#1
0
    def __init__(self, *args, **kwargs):

        super(CTXLinklistRegistry, self).__init__(*args, **kwargs)

        self.manager = Linklist()
        self.events = MongoStorage(table='events')
        self.context = Context()
示例#2
0
    def __init__(self, table=None, *args, **kwargs):
        """
        :param Storage storage: event storage to use.
        """

        super(CTXOldRegistry, self).__init__(*args, **kwargs)

        self.storage = MongoStorage(table=table)
示例#3
0
    def update_mongo2influxdb(self):
        """Convert mongo data to influxdb data."""
        mongostorage = MongoStorage(table='periodic_perfdata')

        count = mongostorage._backend.count()

        if count:  # if data have to be deleted
            for document in mongostorage._backend.find():

                data_id = document['i']
                timestamp = int(document['t'])
                values = document['v']

                points = [(timestamp + int(ts), values[ts]) for ts in values]

                perfdata.put(metric_id=data_id, points=points)

                mongostorage.remove_elements(ids=document['_id'])
示例#4
0
文件: bench.py 项目: crudbug/canopsis
    def setUp(self):
        """
        initalize a storage.
        """

        self.storage = MongoStorage(data_scope='test')
        # self.storage.safe = False
        # remove all data from collection
        self.storage.drop()
        # initialize numerical values
        self.iteration = 5
        self.count = 10000
        # generate count documents
        self.documents = list(
            {
                "_id": str(i),
                "data": {"index": i}
            } for i in range(self.count)
        )
        self.documents_to_update = list(
            {
                "$set": {
                    'data.index': None
                }
            } for i in range(self.count)
        )
        # initialize commands
        self.commands = (
            ("insert", lambda spec, document: self.storage._insert(
                document=document, cache=True)),
            ("update", lambda spec, document: self.storage._update(
                spec=spec, document=document, multi=False, cache=True)),
            ("find", lambda spec, document: self.storage._find(
                document=document, cache=True)),
            ("remove", lambda spec, document: self.storage._remove(
                document=document, multi=True, cache=True))
        )

        self.max_connections = 1
示例#5
0
文件: bench.py 项目: crudbug/canopsis
class Bench(TestCase):

    def setUp(self):
        """
        initalize a storage.
        """

        self.storage = MongoStorage(data_scope='test')
        # self.storage.safe = False
        # remove all data from collection
        self.storage.drop()
        # initialize numerical values
        self.iteration = 5
        self.count = 10000
        # generate count documents
        self.documents = list(
            {
                "_id": str(i),
                "data": {"index": i}
            } for i in range(self.count)
        )
        self.documents_to_update = list(
            {
                "$set": {
                    'data.index': None
                }
            } for i in range(self.count)
        )
        # initialize commands
        self.commands = (
            ("insert", lambda spec, document: self.storage._insert(
                document=document, cache=True)),
            ("update", lambda spec, document: self.storage._update(
                spec=spec, document=document, multi=False, cache=True)),
            ("find", lambda spec, document: self.storage._find(
                document=document, cache=True)),
            ("remove", lambda spec, document: self.storage._remove(
                document=document, multi=True, cache=True))
        )

        self.max_connections = 1

    def tearDown(self):
        """
        End the test
        """

        # remove all data from collection
        self.storage.drop()

    def test(self):
        """
        Run tests.
        """

        threads = [
            Thread(target=self._test_CRUD) for i in range(self.max_connections)
        ]

        for thread in threads:
            thread.start()

        for thread in threads:
            thread.join()

    def _test_CRUD(self):
        """
        Bench CRUD commands
        """

        print(
            'Starting bench on %s with %s documents'
            %
            (self.commands, self.count)
        )

        stats_per_command = {
            command[0]: {'min': None, 'max': None, 'mean': 0}
            for command in self.commands
        }

        min_t, max_t, mean_t = None, None, 0

        for i in range(self.iteration):

            total = 0

            for index, command in enumerate(self.commands):

                fn = command[1]
                command = command[0]

                start = time()

                if command == 'update':
                    documents = self.documents_to_update
                else:
                    documents = self.documents

                for j in range(self.count):

                    fn(
                        spec={'_id': str(self.count)},
                        document=documents[j]
                    )

                stop = time()

                duration = stop - start
                stats = stats_per_command[command]
                if stats['min'] is None or stats['min'] > duration:
                    stats['min'] = duration
                if stats['max'] is None or stats['max'] < duration:
                    stats['max'] = duration
                stats['mean'] += duration

                bench_msg = 'command: %s, iteration: %s, time: %s'

                print(
                    bench_msg % (command, i, duration)
                )

                total += duration

            if min_t is None or min_t > total:
                min_t = total
            if max_t is None or max_t < total:
                max_t = total
            mean_t += total

            bench_msg = 'CRUD: %s' % total

            print(bench_msg)

        # update mean per command
        for command in self.commands:
            stats = stats_per_command[command[0]]
            stats['mean'] = stats['mean'] / self.iteration
            bench_msg = 'command: %s, min: %s, max: %s, mean: %s'
            print(
                bench_msg
                %
                (command[0], stats['min'], stats['max'], stats['mean'])
            )

        # update mean for all CRUD operations
        mean_t = mean_t / self.iteration

        bench_msg = "all command, min: %s, max: %s, mean: %s"
        print(
            bench_msg % (min_t, max_t, mean_t)
        )
示例#6
0
class CTXOldRegistry(CTXPropRegistry):
    """In charge of binding an old collection (not generated from the Storage
    project) to context entities.

    Such old collections contain all entity fields.
    """

    def __init__(self, table=None, *args, **kwargs):
        """
        :param Storage storage: event storage to use.
        """

        super(CTXOldRegistry, self).__init__(*args, **kwargs)

        self.storage = MongoStorage(table=table)

    def _do(self, command, ids, query, queryname, *args, **kwargs):
        """Execute storage command related to input ids and query.

        :param command: storage command.
        :param list ids: entity id(s).
        :param dict query: storage command query.
        :param str queryname: storage command query parameter.
        :return: storage command execution documents.
        :rtype: list
        """

        result = []
        # initialize query
        query = deepcopy(query) if query else {}
        # get entity id field name
        ctx_id_field = self._ctx_id_field()

        if ids is None:
            # get all existing entity ids
            ids = []
            events = self.storage.find_elements()
            for event in events:
                entity = self.context.get_entity(event)
                entity_id = self.context.get_entity_id(entity)
                ids.append(entity_id)

        for entity_id in ids:
            # get entity
            entity = self.context.get_entity_by_id(entity_id)
            cleaned_entity = self.context.clean(entity)
            cleaned_entity['source_type'] = cleaned_entity.pop('type')
            for ctx in self.context.context[1:]:
                if ctx in cleaned_entity:
                    continue
                else:
                    cleaned_entity[ctx] = cleaned_entity.pop(Context.NAME)
                    break
            # update query with entity information
            _query = deepcopy(query)
            _query.setdefault('$and', []).append(cleaned_entity)
            # update kwargs with query and queryname
            kwargs[queryname] = _query
            # execute the storage command
            documents = command(*args, **kwargs)
            if isinstance(documents, Cursor):
                documents = list(documents)
                # update entity_id in documents
                for document in documents:
                    document[ctx_id_field] = entity_id
            else:
                documents = [
                    {ctx_id_field: entity_id, 'result': documents}
                ]
            # add all documents into the result
            result += documents

        return result

    def _get(self, ids, query, *args, **kwargs):

        return self._do(
            command=self.storage.find_elements,
            ids=ids, query=query, queryname='query',
            *args, **kwargs
        )

    def _count(self, ids, query, *args, **kwargs):

        return self._do(
            command=self.storage.count_elements,
            ids=ids, query=query, queryname='query',
            *args, **kwargs
        )

    def _delete(self, ids, query, *args, **kwargs):

        return self._do(
            command=self.storage.remove_elements,
            ids=ids, query=query, queryname='_filter',
            *args, **kwargs
        )

    def ids(self, query=None, *args, **kwargs):

        result = []

        events = self.storage.find_elements(query=query)

        for event in events:
            entity = self.context.get_entity(event)
            entity_id = self.context.get_entity_id(entity)
            result.append(entity_id)

        return result
示例#7
0
class CTXLinklistRegistry(CTXPropRegistry):
    """In charge of ctx linklist information.
    """

    __datatype__ = 'linklist'  #: default datatype name

    def __init__(self, *args, **kwargs):

        super(CTXLinklistRegistry, self).__init__(*args, **kwargs)

        self.manager = Linklist()
        self.events = MongoStorage(table='events')
        self.context = Context()

    def _get_documents(self, ids, query):
        """Get documents related to input ids and query.

        :param list ids: entity ids. If None, get all documents.
        :param dict query: additional selection query.
        :return: list of documents.
        :rtype: list
        """
        result = []
        # get entity id field name
        ctx_id_field = self._ctx_id_field()
        # get a set of entity ids for execution speed reasons
        if ids is not None:
            ids = set(ids)
        # get documents
        docs = self.manager.find(_filter=query)
        for doc in docs:
            try:
                mfilter = loads(doc['mfilter'])
            except Exception:
                pass
            else:  # get entities from events
                events = self.events.find_elements(query=mfilter)
                for event in events:
                    entity = self.context.get_entity(event)
                    entity_id = self.context.get_entity_id(entity)
                    if ids is None or entity_id in ids:
                        doc[ctx_id_field] = entity_id  # add eid to the doc
                        result.append(doc)

        return result

    def _get(self, ids, query, *args, **kwargs):

        return self._get_documents(ids=ids, query=query)

    def _delete(self, ids, query, *args, **kwargs):

        result = self._get_documents(ids=ids, query=query)

        ids = [doc['_id'] for doc in result]

        self.manager.remove(ids=ids)

        return result

    def ids(self, query=None):

        result = []

        documents = self.manager.find(_filter=query)

        for document in documents:
            try:
                mfilter = loads(document['mfilter'])
            except Exception:
                pass
            else:
                # get entities from events
                events = self.events.find_elements(query=mfilter)
                for event in events:
                    entity = self.context.get_entity(event)
                    entity_id = self.context.get_entity_id(entity)
                    result.append(entity_id)

        return result