Esempio n. 1
0
    def _get_from_logs_cf(self, columns_keys):
        if not columns_keys:
            return []
        result = []
        mapped_row_keys = {}
        for a_column_key in columns_keys:
            row_key = ymd_from_uuid1(a_column_key)
            if row_key in mapped_row_keys:
                mapped_row_keys[row_key].append(a_column_key)
            else:
                mapped_row_keys[row_key] = [a_column_key]

        for a_row_key in sorted(mapped_row_keys.keys(), reverse=True):
            cass_result_from_logs = self._get_cf_logs().get(
                a_row_key, columns=mapped_row_keys[a_row_key])
            #ordered_result = [(val[1], key, val[0], ) for key, val in cass_result_from_logs.iteritems()]
            #ordered_result = sorted(ordered_result, reverse=True)
            #result = result + [json.loads(col_val) for _, _, col_val in ordered_result]
            result = result + [
                json.loads(col_val)
                for _, col_val in cass_result_from_logs.iteritems()
            ]

        return sorted(result,
                      key=lambda msg: float(msg['timestamp']),
                      reverse=True)
Esempio n. 2
0
 def get_by_id(self, message_id):
     # FIXME: add documentation
     # FIXME: add tests
     # FIXME: return None if doesn't exists
     msg_uuid = uuid.UUID(hex=message_id)
     row_key = ymd_from_uuid1(msg_uuid)
     json_str = self._get_cf_logs().get(row_key, columns=[msg_uuid])[msg_uuid]
     return json.loads(json_str)
Esempio n. 3
0
 def get_by_id(self, message_id):
     # FIXME: add documentation
     # FIXME: add tests
     # FIXME: return None if doesn't exists
     msg_uuid = uuid.UUID(hex=message_id)
     row_key = ymd_from_uuid1(msg_uuid)
     json_str = self._get_cf_logs().get(row_key,
                                        columns=[msg_uuid])[msg_uuid]
     return json.loads(json_str)
Esempio n. 4
0
    def save_log(self, application, host, severity, timestamp, message):
        """
        Saves a log message.
        Raises:
        - DaedalusException if any parameter isn't valid.
        """
        _check_application(application)
        _check_severity(severity)
        _check_host(host)
        _check_message(message)
        try:
            timestamp = float(timestamp)
        except:
            raise(DaedalusException("The timestamp '{0}' couldn't be transformed to a float".format(timestamp)))

        event_uuid = convert_time_to_uuid(timestamp, randomize=True)
        _id = event_uuid.get_hex()

        json_message = json.dumps({
            'application': application,
            'host': host,
            'severity': severity,
            'timestamp': timestamp,
            '_id': _id,
            'message': message,
        })

        pool = self._get_pool()
        with Mutator(pool) as batch:
            # Save on <CF> CF_LOGS
            row_key = ymd_from_uuid1(event_uuid)
            batch.insert(
                self._get_cf_logs(),
                str(row_key), {
                    event_uuid: json_message,
            })

            # Save on <CF> CF_LOGS_BY_APP
            batch.insert(
                self._get_cf_logs_by_app(),
                application, {
                    event_uuid: EMPTY_VALUE,
            })

            # Save on <CF> CF_LOGS_BY_HOST
            batch.insert(
                self._get_cf_logs_by_host(),
                host, {
                    event_uuid: EMPTY_VALUE,
            })

            # Save on <CF> CF_LOGS_BY_SEVERITY
            batch.insert(
                self._get_cf_logs_by_severity(),
                severity, {
                    event_uuid: EMPTY_VALUE,
            })
Esempio n. 5
0
def mass_insert(pool):
    cf_logs = ColumnFamily(pool, CF_LOGS)
    cf_logs_by_app = ColumnFamily(pool, CF_LOGS_BY_APP)
    cf_logs_by_host = ColumnFamily(pool, CF_LOGS_BY_HOST)
    cf_logs_by_severity = ColumnFamily(pool, CF_LOGS_BY_SEVERITY)
    rnd_inst = random.Random()
    rnd_inst.seed(1)
    start = time.time()
    count = 0
    try:
        for item in log_generator(1):
            msg = item[0]
            app = item[1]
            host = item[2]
            severity = item[3]

            # http://pycassa.github.com/pycassa/assorted/time_uuid.html
            # http://www.slideshare.net/jeremiahdjordan/pycon-2012-apache-cassandra
            # http://www.slideshare.net/rbranson/how-do-i-cassandra @ slide 80
            # https://github.com/pycassa/pycassa/issues/135

            # Save on <CF> CF_LOGS
            event_uuid = uuid.uuid1()
            row_key = ymd_from_uuid1(event_uuid)
            cf_logs.insert(str(row_key), {
                event_uuid: msg,
            })

            # Save on <CF> CF_LOGS_BY_APP
            cf_logs_by_app.insert(app, {
                event_uuid: msg,
            })

            # Save on <CF> CF_LOGS_BY_HOST
            cf_logs_by_host.insert(host, {
                event_uuid: msg,
            })

            # Save on <CF> CF_LOGS_BY_SEVERITY
            cf_logs_by_severity.insert(severity, {
                event_uuid: msg,
            })

            count += 4
            if count % 400 == 0:
                avg = float(count) / (time.time() - start)
                logging.info("Inserted %d columns, %f insert/sec", count, avg)
    except KeyboardInterrupt:
        logging.info("Stopping...")
    end = time.time()
    avg = float(count) / (end - start)
    logging.info("%d columns inserted. Avg: %f insert/sec", count, avg)
Esempio n. 6
0
    def save_log(self, application, host, severity, timestamp, message):
        """
        Saves a log message.
        Raises:
        - DaedalusException if any parameter isn't valid.
        """
        _check_application(application)
        _check_severity(severity)
        _check_host(host)
        _check_message(message)
        try:
            timestamp = float(timestamp)
        except:
            raise (DaedalusException(
                "The timestamp '{0}' couldn't be transformed to a float".
                format(timestamp)))

        event_uuid = convert_time_to_uuid(timestamp, randomize=True)
        _id = event_uuid.get_hex()

        json_message = json.dumps({
            'application': application,
            'host': host,
            'severity': severity,
            'timestamp': timestamp,
            '_id': _id,
            'message': message,
        })

        pool = self._get_pool()
        with Mutator(pool) as batch:
            # Save on <CF> CF_LOGS
            row_key = ymd_from_uuid1(event_uuid)
            batch.insert(self._get_cf_logs(), str(row_key), {
                event_uuid: json_message,
            })

            # Save on <CF> CF_LOGS_BY_APP
            batch.insert(self._get_cf_logs_by_app(), application, {
                event_uuid: EMPTY_VALUE,
            })

            # Save on <CF> CF_LOGS_BY_HOST
            batch.insert(self._get_cf_logs_by_host(), host, {
                event_uuid: EMPTY_VALUE,
            })

            # Save on <CF> CF_LOGS_BY_SEVERITY
            batch.insert(self._get_cf_logs_by_severity(), severity, {
                event_uuid: EMPTY_VALUE,
            })
Esempio n. 7
0
    def _get_from_logs_cf(self, columns_keys):
        if not columns_keys:
            return []
        result = []
        mapped_row_keys = {}
        for a_column_key in columns_keys:
            row_key = ymd_from_uuid1(a_column_key)
            if row_key in mapped_row_keys:
                mapped_row_keys[row_key].append(a_column_key)
            else:
                mapped_row_keys[row_key] = [a_column_key]

        for a_row_key in sorted(mapped_row_keys.keys(), reverse=True):
            cass_result_from_logs = self._get_cf_logs().get(a_row_key, columns=mapped_row_keys[a_row_key])
            #ordered_result = [(val[1], key, val[0], ) for key, val in cass_result_from_logs.iteritems()]
            #ordered_result = sorted(ordered_result, reverse=True)
            #result = result + [json.loads(col_val) for _, _, col_val in ordered_result]
            result = result + [json.loads(col_val) for _, col_val in cass_result_from_logs.iteritems()]

        return sorted(result, key=lambda msg: float(msg['timestamp']), reverse=True)