def _get_from_logs_cf(self, columns_keys): if not columns_keys: return [] result = [] mapped_row_keys = {} for a_column_key in columns_keys: row_key = ymd_from_uuid1(a_column_key) if row_key in mapped_row_keys: mapped_row_keys[row_key].append(a_column_key) else: mapped_row_keys[row_key] = [a_column_key] for a_row_key in sorted(mapped_row_keys.keys(), reverse=True): cass_result_from_logs = self._get_cf_logs().get( a_row_key, columns=mapped_row_keys[a_row_key]) #ordered_result = [(val[1], key, val[0], ) for key, val in cass_result_from_logs.iteritems()] #ordered_result = sorted(ordered_result, reverse=True) #result = result + [json.loads(col_val) for _, _, col_val in ordered_result] result = result + [ json.loads(col_val) for _, col_val in cass_result_from_logs.iteritems() ] return sorted(result, key=lambda msg: float(msg['timestamp']), reverse=True)
def get_by_id(self, message_id): # FIXME: add documentation # FIXME: add tests # FIXME: return None if doesn't exists msg_uuid = uuid.UUID(hex=message_id) row_key = ymd_from_uuid1(msg_uuid) json_str = self._get_cf_logs().get(row_key, columns=[msg_uuid])[msg_uuid] return json.loads(json_str)
def save_log(self, application, host, severity, timestamp, message): """ Saves a log message. Raises: - DaedalusException if any parameter isn't valid. """ _check_application(application) _check_severity(severity) _check_host(host) _check_message(message) try: timestamp = float(timestamp) except: raise(DaedalusException("The timestamp '{0}' couldn't be transformed to a float".format(timestamp))) event_uuid = convert_time_to_uuid(timestamp, randomize=True) _id = event_uuid.get_hex() json_message = json.dumps({ 'application': application, 'host': host, 'severity': severity, 'timestamp': timestamp, '_id': _id, 'message': message, }) pool = self._get_pool() with Mutator(pool) as batch: # Save on <CF> CF_LOGS row_key = ymd_from_uuid1(event_uuid) batch.insert( self._get_cf_logs(), str(row_key), { event_uuid: json_message, }) # Save on <CF> CF_LOGS_BY_APP batch.insert( self._get_cf_logs_by_app(), application, { event_uuid: EMPTY_VALUE, }) # Save on <CF> CF_LOGS_BY_HOST batch.insert( self._get_cf_logs_by_host(), host, { event_uuid: EMPTY_VALUE, }) # Save on <CF> CF_LOGS_BY_SEVERITY batch.insert( self._get_cf_logs_by_severity(), severity, { event_uuid: EMPTY_VALUE, })
def mass_insert(pool): cf_logs = ColumnFamily(pool, CF_LOGS) cf_logs_by_app = ColumnFamily(pool, CF_LOGS_BY_APP) cf_logs_by_host = ColumnFamily(pool, CF_LOGS_BY_HOST) cf_logs_by_severity = ColumnFamily(pool, CF_LOGS_BY_SEVERITY) rnd_inst = random.Random() rnd_inst.seed(1) start = time.time() count = 0 try: for item in log_generator(1): msg = item[0] app = item[1] host = item[2] severity = item[3] # http://pycassa.github.com/pycassa/assorted/time_uuid.html # http://www.slideshare.net/jeremiahdjordan/pycon-2012-apache-cassandra # http://www.slideshare.net/rbranson/how-do-i-cassandra @ slide 80 # https://github.com/pycassa/pycassa/issues/135 # Save on <CF> CF_LOGS event_uuid = uuid.uuid1() row_key = ymd_from_uuid1(event_uuid) cf_logs.insert(str(row_key), { event_uuid: msg, }) # Save on <CF> CF_LOGS_BY_APP cf_logs_by_app.insert(app, { event_uuid: msg, }) # Save on <CF> CF_LOGS_BY_HOST cf_logs_by_host.insert(host, { event_uuid: msg, }) # Save on <CF> CF_LOGS_BY_SEVERITY cf_logs_by_severity.insert(severity, { event_uuid: msg, }) count += 4 if count % 400 == 0: avg = float(count) / (time.time() - start) logging.info("Inserted %d columns, %f insert/sec", count, avg) except KeyboardInterrupt: logging.info("Stopping...") end = time.time() avg = float(count) / (end - start) logging.info("%d columns inserted. Avg: %f insert/sec", count, avg)
def save_log(self, application, host, severity, timestamp, message): """ Saves a log message. Raises: - DaedalusException if any parameter isn't valid. """ _check_application(application) _check_severity(severity) _check_host(host) _check_message(message) try: timestamp = float(timestamp) except: raise (DaedalusException( "The timestamp '{0}' couldn't be transformed to a float". format(timestamp))) event_uuid = convert_time_to_uuid(timestamp, randomize=True) _id = event_uuid.get_hex() json_message = json.dumps({ 'application': application, 'host': host, 'severity': severity, 'timestamp': timestamp, '_id': _id, 'message': message, }) pool = self._get_pool() with Mutator(pool) as batch: # Save on <CF> CF_LOGS row_key = ymd_from_uuid1(event_uuid) batch.insert(self._get_cf_logs(), str(row_key), { event_uuid: json_message, }) # Save on <CF> CF_LOGS_BY_APP batch.insert(self._get_cf_logs_by_app(), application, { event_uuid: EMPTY_VALUE, }) # Save on <CF> CF_LOGS_BY_HOST batch.insert(self._get_cf_logs_by_host(), host, { event_uuid: EMPTY_VALUE, }) # Save on <CF> CF_LOGS_BY_SEVERITY batch.insert(self._get_cf_logs_by_severity(), severity, { event_uuid: EMPTY_VALUE, })
def _get_from_logs_cf(self, columns_keys): if not columns_keys: return [] result = [] mapped_row_keys = {} for a_column_key in columns_keys: row_key = ymd_from_uuid1(a_column_key) if row_key in mapped_row_keys: mapped_row_keys[row_key].append(a_column_key) else: mapped_row_keys[row_key] = [a_column_key] for a_row_key in sorted(mapped_row_keys.keys(), reverse=True): cass_result_from_logs = self._get_cf_logs().get(a_row_key, columns=mapped_row_keys[a_row_key]) #ordered_result = [(val[1], key, val[0], ) for key, val in cass_result_from_logs.iteritems()] #ordered_result = sorted(ordered_result, reverse=True) #result = result + [json.loads(col_val) for _, _, col_val in ordered_result] result = result + [json.loads(col_val) for _, col_val in cass_result_from_logs.iteritems()] return sorted(result, key=lambda msg: float(msg['timestamp']), reverse=True)