Example #1
0
def mass_insert(pool):
    cf_logs = ColumnFamily(pool, CF_LOGS)
    rnd_inst = random.Random()
    rnd_inst.seed(1)
    start = time.time()
    count = 0
    try:
        for item in log_generator(1):
            msg = item[0]
            app = item[1]

            # http://pycassa.github.com/pycassa/assorted/time_uuid.html
            # http://www.slideshare.net/jeremiahdjordan/pycon-2012-apache-cassandra
            # http://www.slideshare.net/rbranson/how-do-i-cassandra @ slide 80
            # https://github.com/pycassa/pycassa/issues/135
            cf_logs.insert(app, {
                uuid.uuid1(): msg,
            })
            count += 1
            if count % 100 == 0:
                logging.info("Inserted %d columns", count)
    except KeyboardInterrupt:
        logging.info("Stopping...")
    end = time.time()
    avg = float(count) / (end - start)
    logging.info("Avg: %f insert/sec", avg)
Example #2
0
def mass_insert(pool):
    cf_logs = ColumnFamily(pool, CF_LOGS)
    cf_logs_by_app = ColumnFamily(pool, CF_LOGS_BY_APP)
    cf_logs_by_host = ColumnFamily(pool, CF_LOGS_BY_HOST)
    cf_logs_by_severity = ColumnFamily(pool, CF_LOGS_BY_SEVERITY)
    rnd_inst = random.Random()
    rnd_inst.seed(1)
    start = time.time()
    count = 0
    try:
        for item in log_generator(1):
            msg = item[0]
            app = item[1]
            host = item[2]
            severity = item[3]

            # http://pycassa.github.com/pycassa/assorted/time_uuid.html
            # http://www.slideshare.net/jeremiahdjordan/pycon-2012-apache-cassandra
            # http://www.slideshare.net/rbranson/how-do-i-cassandra @ slide 80
            # https://github.com/pycassa/pycassa/issues/135

            # Save on <CF> CF_LOGS
            event_uuid = uuid.uuid1()
            row_key = ymd_from_uuid1(event_uuid)
            cf_logs.insert(str(row_key), {
                event_uuid: msg,
            })

            # Save on <CF> CF_LOGS_BY_APP
            cf_logs_by_app.insert(app, {
                event_uuid: msg,
            })

            # Save on <CF> CF_LOGS_BY_HOST
            cf_logs_by_host.insert(host, {
                event_uuid: msg,
            })

            # Save on <CF> CF_LOGS_BY_SEVERITY
            cf_logs_by_severity.insert(severity, {
                event_uuid: msg,
            })

            count += 4
            if count % 400 == 0:
                avg = float(count) / (time.time() - start)
                logging.info("Inserted %d columns, %f insert/sec", count, avg)
    except KeyboardInterrupt:
        logging.info("Stopping...")
    end = time.time()
    avg = float(count) / (end - start)
    logging.info("%d columns inserted. Avg: %f insert/sec", count, avg)
Example #3
0
def mass_insert(pool):
    cf_logs = ColumnFamily(pool, CF_LOGS)
    rnd_inst = random.Random()
    rnd_inst.seed(1)
    start = time.time()
    count = 0
    try:
        for item in log_generator(1):
            msg = item[0]
            app = item[1]

            # http://pycassa.github.com/pycassa/assorted/time_uuid.html
            # http://www.slideshare.net/jeremiahdjordan/pycon-2012-apache-cassandra
            # http://www.slideshare.net/rbranson/how-do-i-cassandra @ slide 80
            # https://github.com/pycassa/pycassa/issues/135
            cf_logs.insert(app, {uuid.uuid1(): msg})
            count += 1
            if count % 100 == 0:
                logging.info("Inserted %d columns", count)
    except KeyboardInterrupt:
        logging.info("Stopping...")
    end = time.time()
    avg = float(count) / (end - start)
    logging.info("Avg: %f insert/sec", avg)