class TestCustomTypes(unittest.TestCase):
class IntString(types.CassandraType):
@staticmethod
def pack(intval):
return str(intval)
@staticmethod
def unpack(strval):
return int(strval)
class IntString2(types.CassandraType):
def __init__(self, *args, **kwargs):
self.pack = lambda val: str(val)
self.unpack = lambda val: int(val)
def test_staticmethod_funcs(self):
self.cf = ColumnFamily(pool, 'Standard1')
self.cf.key_validation_class = TestCustomTypes.IntString()
self.cf.insert(1234, {'col': 'val'})
assert_equal(self.cf.get(1234), {'col': 'val'})
def test_constructor_lambdas(self):
self.cf = ColumnFamily(pool, 'Standard1')
self.cf.key_validation_class = TestCustomTypes.IntString2()
self.cf.insert(1234, {'col': 'val'})
assert_equal(self.cf.get(1234), {'col': 'val'})
def test_alter_column_non_bytes_type(self):
sys.create_column_family(TEST_KS, 'LongCF', comparator_type=LONG_TYPE)
sys.create_index(TEST_KS, 'LongCF', 3, LONG_TYPE)
pool = ConnectionPool(TEST_KS)
cf = ColumnFamily(pool, 'LongCF')
cf.insert('key', {3: 3})
assert_equal(cf.get('key')[3], 3)
sys.alter_column(TEST_KS, 'LongCF', 2, LONG_TYPE)
cf = ColumnFamily(pool, 'LongCF')
cf.insert('key', {2: 2})
assert_equal(cf.get('key')[2], 2)
def test_alter_column_non_bytes_type(self):
sys.create_column_family(TEST_KS, 'LongCF', comparator_type=LONG_TYPE)
sys.create_index(TEST_KS, 'LongCF', 3, LONG_TYPE)
pool = ConnectionPool(TEST_KS)
cf = ColumnFamily(pool, 'LongCF')
cf.insert('key', {3: 3})
assert_equal(cf.get('key')[3], 3)
sys.alter_column(TEST_KS, 'LongCF', 2, LONG_TYPE)
cf = ColumnFamily(pool, 'LongCF')
cf.insert('key', {2: 2})
assert_equal(cf.get('key')[2], 2)
def test_column_validators(self):
validators = {'name': UTF8_TYPE, 'age': LONG_TYPE}
sys.create_column_family(TEST_KS, 'ValidatedCF',
column_validation_classes=validators)
pool = ConnectionPool(TEST_KS)
cf = ColumnFamily(pool, 'ValidatedCF')
cf.insert('key', {'name': 'John', 'age': 40})
self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
validators = {'name': ASCII_TYPE, 'age': INT_TYPE}
sys.alter_column_family(TEST_KS, 'ValidatedCF',
column_validation_classes=validators)
cf.load_schema()
self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
def test_static_composite_slicing(self):
cf = ColumnFamily(pool, 'StaticComposite')
u1 = uuid.uuid1()
u4 = uuid.uuid4()
col0 = (0, 1, u1, u4, '', '', '')
col1 = (1, 1, u1, u4, '', '', '')
col2 = (1, 2, u1, u4, '', '', '')
col3 = (1, 3, u1, u4, '', '', '')
col4 = (2, 1, u1, u4, '', '', '')
cf.insert('key2', {col0: '', col1: '', col2: '', col3: '', col4: ''})
result = cf.get('key2', column_start=((1, True),), column_finish=((1, True),))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1,), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=((1, True),), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, ), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=((0, False), ), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, 1), column_finish=(1, 3))
assert_equal(result, {col1: '', col2: ''})
result = cf.get('key2', column_start=(1, 1), column_finish=(1, (3, True)))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, (1, True)), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
def test_static_composite_slicing(self):
cf = ColumnFamily(pool, 'StaticComposite')
u1 = uuid.uuid1()
u4 = uuid.uuid4()
col0 = (0, 1, u1, u4, '', '', '')
col1 = (1, 1, u1, u4, '', '', '')
col2 = (1, 2, u1, u4, '', '', '')
col3 = (1, 3, u1, u4, '', '', '')
col4 = (2, 1, u1, u4, '', '', '')
cf.insert('key2', {col0: '', col1: '', col2: '', col3: '', col4: ''})
result = cf.get('key2', column_start=((1, True),), column_finish=((1, True),))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1,), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=((1, True),), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, ), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=((0, False), ), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, 1), column_finish=(1, 3))
assert_equal(result, {col1: '', col2: ''})
result = cf.get('key2', column_start=(1, 1), column_finish=(1, (3, True)))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, (1, True)), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
def test_static_composite_slicing(self):
cf = ColumnFamily(pool, "StaticComposite")
u1 = uuid.uuid1()
u4 = uuid.uuid4()
col0 = (0, 1, u1, u4, "", "", "")
col1 = (1, 1, u1, u4, "", "", "")
col2 = (1, 2, u1, u4, "", "", "")
col3 = (1, 3, u1, u4, "", "", "")
col4 = (2, 1, u1, u4, "", "", "")
cf.insert("key2", {col0: "", col1: "", col2: "", col3: "", col4: ""})
result = cf.get("key2", column_start=((1, True),), column_finish=((1, True),))
assert_equal(result, {col1: "", col2: "", col3: ""})
result = cf.get("key2", column_start=(1,), column_finish=((2, False),))
assert_equal(result, {col1: "", col2: "", col3: ""})
result = cf.get("key2", column_start=((1, True),), column_finish=((2, False),))
assert_equal(result, {col1: "", col2: "", col3: ""})
result = cf.get("key2", column_start=(1,), column_finish=((2, False),))
assert_equal(result, {col1: "", col2: "", col3: ""})
result = cf.get("key2", column_start=((0, False),), column_finish=((2, False),))
assert_equal(result, {col1: "", col2: "", col3: ""})
result = cf.get("key2", column_start=(1, 1), column_finish=(1, 3))
assert_equal(result, {col1: "", col2: "", col3: ""})
result = cf.get("key2", column_start=(1, 1), column_finish=(1, (3, True)))
assert_equal(result, {col1: "", col2: "", col3: ""})
result = cf.get("key2", column_start=(1, (1, True)), column_finish=((2, False),))
assert_equal(result, {col1: "", col2: "", col3: ""})
def test_static_composite(cls):
sys = SystemManager()
have_composites = sys._conn.version != CASSANDRA_07
if not have_composites:
raise SkipTest("Cassandra < 0.8 does not composite types")
sys.create_column_family(TEST_KS, 'StaticComposite',
comparator_type=CompositeType(LongType(),
IntegerType(),
TimeUUIDType(reversed=True),
LexicalUUIDType(reversed=False),
AsciiType(),
UTF8Type(),
BytesType()))
cf = ColumnFamily(pool, 'StaticComposite')
colname = (127312831239123123, 1, uuid.uuid1(), uuid.uuid4(), 'foo', u'ba\u0254r', 'baz')
cf.insert('key', {colname: 'val'})
assert_equal(cf.get('key'), {colname: 'val'})
u1 = uuid.uuid1()
u4 = uuid.uuid4()
col0 = (0, 1, u1, u4, '', '', '')
col1 = (1, 1, u1, u4, '', '', '')
col2 = (1, 2, u1, u4, '', '', '')
col3 = (1, 3, u1, u4, '', '', '')
col4 = (2, 1, u1, u4, '', '', '')
cf.insert('key2', {col0: '', col1: '', col2: '', col3: '', col4: ''})
result = cf.get('key2', column_start=((1, True),), column_finish=((1, True),))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1,), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=((1, True),), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, ), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=((0, False), ), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, 1), column_finish=(1, 3))
assert_equal(result, {col1: '', col2: ''})
result = cf.get('key2', column_start=(1, 1), column_finish=(1, (3, True)))
assert_equal(result, {col1: '', col2: '', col3: ''})
result = cf.get('key2', column_start=(1, (1, True)), column_finish=((2, False), ))
assert_equal(result, {col1: '', col2: '', col3: ''})
sys.drop_column_family(TEST_KS, 'StaticComposite')
class Processor(Llama):
def __init__(self, client, qname):
super(Processor, self).__init__(client, qname)
self.pool = ConnectionPool('processing_llama_Processor')
self.trends = ColumnFamily(self.pool, 'Trend')
def get_sleep_time():
return 60
def do_message(self, message):
if not isinstance(message, tuple) or len(message) != 4:
return
woeid, as_of, trend_name, query = message
try:
trend = self.trends.get(trend_name, super_column=woeid)
trend['lastseen'] = as_of
trend['number_seen'] += 1
trend = { woeid: trend }
self.trends.insert(trend_name, trend)
except ttypes.NotFoundException:
self.trends.insert(trend_name, { woeid: { 'firstseen': as_of, 'lastseen': as_of, 'number_seen': 1}})
self.trends.insert(trend_name, {'data': { 'query': query, 'tracking': "False" }})
def has_many_woeids(self, x):
key, values = x
return len(values) > 2 # one woeid plus general data
def do_action(self):
try:
for trend_name, country_specifics in filter(self.has_many_woeids, self.trends.get_range()):
if country_specifics['data']['tracking'] == "False":
self.track_trend(trend_name, country_specifics['data']['query'], filter(lambda x: x != "data", country_specifics.keys()))
except:
exc, value, exctraceback = sys.exc_info()
logging.error("Error in processing_llama.processor.Processor.do_action:\n" + traceback.format_exc())
traceback.print_tb(exctraceback)
def track_trend(self, trend_name, query, woeids):
logging.info("Tracking %s from %s" % (trend_name, woeids))
self.mark_tracking(trend_name)
self.publish((trend_name, query, woeids), "trend_to_track")
def mark_tracking(self, trend_name):
try:
trend = self.trends.get(trend_name, super_column='data')
trend['tracking'] = "True"
self.trends.insert(trend_name, { 'data': trend })
except ttypes.NotFoundException:
pass
def get_row_key_id(domain):
counter_column, counter_lock = domain_counter_map[domain]
## acquire lock before getting value of
counter_lock.acquire()
try:
client = db_connection.get_client()
cf = ColumnFamily(client, CONFIG_DOMAIN)
## get new key id
id_key = cf.get(CONFIG_ROW, counter_column)[counter_column]
## increment value if not None
if id_key:
new_id_key = id_key + 1
cf.insert(CONFIG_ROW, {counter_column: new_id_key}, write_consistency_level=ConsistencyLevel.ALL)
return id_key
"""
if id_key:
str_id_key = str(id_key)
str_id_key.zfill(MAX_PADDING_RANGE)
return str_id_key
else:
return None
"""
finally:
## release lock before returning from this function
counter_lock.release()
class CassandraRepository(object):
def __init__(self, keyspace, column_family_name):
self.pool = ConnectionPool(keyspace, cassandra_settings.NODE_POOL)
self.cf = ColumnFamily(self.pool, column_family_name)
self.batch = {}
def add_batch(self, batch, start_time=None):
"""
:param batch:
"""
self.cf.batch_insert(batch)
if start_time is not None:
print 'time to insert batch: %s ms' % (int(time.time() * 1000) -
start_time)
def get(self, timestamp):
return self.cf.get(str(timestamp))
def get_range(self, start, end):
return list(self.cf.get_range(start=str(start), finish=str(end)))
def close(self):
self.sys.close()
class CassandraRepository(object):
def __init__(self, keyspace, column_family_name):
self.pool = ConnectionPool(keyspace,
cassandra_settings.NODE_POOL)
self.cf = ColumnFamily(self.pool, column_family_name)
self.batch = {}
def add_batch(self, batch, start_time=None):
"""
:param batch:
"""
self.cf.batch_insert(batch)
if start_time is not None:
print 'time to insert batch: %s ms' % (int(time.time() * 1000) - start_time)
def get(self, timestamp):
return self.cf.get(str(timestamp))
def get_range(self, start, end):
return list(self.cf.get_range(start=str(start), finish=str(end)))
def close(self):
self.sys.close()
def get_reverse(columnFamily, key) :
"select values in a given column family"
try :
column = ColumnFamily(pool, columnFamily)
return column.get(key,column_reversed=True)
except:
return {'status':0}
def _get_analytics_start_time(self):
try:
col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
row = col_family.get(SYSTEM_OBJECT_ANALYTICS)
except Exception as e:
self._logger.error("Exception: analytics_start_time Failure %s" %
e)
return None
# Initialize the dictionary before returning
if (SYSTEM_OBJECT_START_TIME not in row):
return None
ret_row = {}
ret_row[SYSTEM_OBJECT_START_TIME] = row[SYSTEM_OBJECT_START_TIME]
if (SYSTEM_OBJECT_FLOW_START_TIME not in row):
ret_row[SYSTEM_OBJECT_FLOW_START_TIME] = row[
SYSTEM_OBJECT_START_TIME]
else:
ret_row[SYSTEM_OBJECT_FLOW_START_TIME] = row[
SYSTEM_OBJECT_FLOW_START_TIME]
if (SYSTEM_OBJECT_STAT_START_TIME not in row):
ret_row[SYSTEM_OBJECT_STAT_START_TIME] = row[
SYSTEM_OBJECT_START_TIME]
else:
ret_row[SYSTEM_OBJECT_STAT_START_TIME] = row[
SYSTEM_OBJECT_STAT_START_TIME]
if (SYSTEM_OBJECT_MSG_START_TIME not in row):
ret_row[SYSTEM_OBJECT_MSG_START_TIME] = row[
SYSTEM_OBJECT_START_TIME]
else:
ret_row[SYSTEM_OBJECT_MSG_START_TIME] = row[
SYSTEM_OBJECT_MSG_START_TIME]
return ret_row
def test_column_validators(self):
validators = {'name': UTF8_TYPE, 'age': LONG_TYPE}
sys.create_column_family(TEST_KS,
'ValidatedCF',
column_validation_classes=validators)
pool = ConnectionPool(TEST_KS)
cf = ColumnFamily(pool, 'ValidatedCF')
cf.insert('key', {'name': 'John', 'age': 40})
self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
validators = {'name': ASCII_TYPE, 'age': INT_TYPE}
sys.alter_column_family(TEST_KS,
'ValidatedCF',
column_validation_classes=validators)
cf.load_schema()
self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
def _get_analytics_start_time(self):
try:
col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
row = col_family.get(SYSTEM_OBJECT_ANALYTICS)
except Exception as e:
self._logger.error("Exception: analytics_start_time Failure %s" % e)
return None
# Initialize the dictionary before returning
if (SYSTEM_OBJECT_START_TIME not in row):
return None
ret_row = {}
ret_row[SYSTEM_OBJECT_START_TIME] = row[SYSTEM_OBJECT_START_TIME]
if (SYSTEM_OBJECT_FLOW_START_TIME not in row):
ret_row[SYSTEM_OBJECT_FLOW_START_TIME] = row[SYSTEM_OBJECT_START_TIME]
else:
ret_row[SYSTEM_OBJECT_FLOW_START_TIME] = row[SYSTEM_OBJECT_FLOW_START_TIME]
if (SYSTEM_OBJECT_STAT_START_TIME not in row):
ret_row[SYSTEM_OBJECT_STAT_START_TIME] = row[SYSTEM_OBJECT_START_TIME]
else:
ret_row[SYSTEM_OBJECT_STAT_START_TIME] = row[SYSTEM_OBJECT_STAT_START_TIME]
if (SYSTEM_OBJECT_MSG_START_TIME not in row):
ret_row[SYSTEM_OBJECT_MSG_START_TIME] = row[SYSTEM_OBJECT_START_TIME]
else:
ret_row[SYSTEM_OBJECT_MSG_START_TIME] = row[SYSTEM_OBJECT_MSG_START_TIME]
return ret_row
def _get_analytics_ttls(self):
ret_row = {}
try:
col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
row = col_family.get(SYSTEM_OBJECT_ANALYTICS)
except Exception as e:
self._logger.error("Exception: analytics_start_time Failure %s" % e)
ret_row[SYSTEM_OBJECT_FLOW_DATA_TTL] = AnalyticsFlowTTL
ret_row[SYSTEM_OBJECT_STATS_DATA_TTL] = AnalyticsStatisticsTTL
ret_row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] = AnalyticsConfigAuditTTL
ret_row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] = AnalyticsTTL
return ret_row
if (SYSTEM_OBJECT_FLOW_DATA_TTL not in row):
ret_row[SYSTEM_OBJECT_FLOW_DATA_TTL] = AnalyticsFlowTTL
else:
ret_row[SYSTEM_OBJECT_FLOW_DATA_TTL] = row[SYSTEM_OBJECT_FLOW_DATA_TTL]
if (SYSTEM_OBJECT_STATS_DATA_TTL not in row):
ret_row[SYSTEM_OBJECT_STATS_DATA_TTL] = AnalyticsStatisticsTTL
else:
ret_row[SYSTEM_OBJECT_STATS_DATA_TTL] = row[SYSTEM_OBJECT_STATS_DATA_TTL]
if (SYSTEM_OBJECT_CONFIG_AUDIT_TTL not in row):
ret_row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] = AnalyticsConfigAuditTTL
else:
ret_row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] = row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL]
if (SYSTEM_OBJECT_GLOBAL_DATA_TTL not in row):
ret_row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] = AnalyticsTTL
else:
ret_row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] = row[SYSTEM_OBJECT_GLOBAL_DATA_TTL]
return ret_row
def build_from_id(self, build_id):
"""Obtain information about a build from its ID."""
cf = ColumnFamily(self.pool, 'builds')
try:
return cf.get(build_id)
except NotFoundException:
return None
def get_builder(self, builder_id):
"""Obtain info about a builder from its ID."""
cf = ColumnFamily(self.pool, 'builders')
try:
return cf.get(builder_id)
except NotFoundException:
return None
def get_row_key_id(domain):
counter_column, counter_lock = domain_counter_map[domain]
## acquire lock before getting value of
counter_lock.acquire()
try:
client = db_connection.get_client()
cf = ColumnFamily(client, CONFIG_DOMAIN)
## get new key id
id_key = cf.get(CONFIG_ROW, counter_column)[counter_column]
## increment value if not None
if id_key:
new_id_key = id_key + 1
cf.insert(CONFIG_ROW, {counter_column: new_id_key},
write_consistency_level=ConsistencyLevel.ALL)
return id_key
"""
if id_key:
str_id_key = str(id_key)
str_id_key.zfill(MAX_PADDING_RANGE)
return str_id_key
else:
return None
"""
finally:
## release lock before returning from this function
counter_lock.release()
def getByTag(tag):
con = util.getConnection()
vidCF = ColumnFamily(con, 'videos')
tagCF = ColumnFamily(con, 'tag_videos')
movies = tagCF.get(tag.strip().lower())
for key, val in movies.iteritems():
vidId = key.split('_')[1]
movieDetail = vidCF.get(vidId)
print '''
{{
user: {0},
movie: {1},
tags: {2}
}}'''.format(movieDetail['user_name'], movieDetail['title'], movieDetail['tags_csv'])
con.dispose()
def show(query, cass):
my_column_family = ColumnFamily(cass, 'my_column_family')
try:
result = my_column_family.get(1167864277)
return result
except NotFoundException, nfe:
return HTTPError(404, 'Entity not found.')
def test_single_component_composite(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'SingleComposite',
comparator_type=CompositeType(IntegerType()))
cf = ColumnFamily(pool, 'SingleComposite')
cf.insert('key', {(123456,): 'val'})
assert_equal(cf.get('key'), {(123456,): 'val'})
def get_one_by_rowkey(self, rowkey, **kwargs):
"""Get the object by the rowkey. Supports pycassa method `get` kwargs."""
col_fam = ColumnFamily(self.pool, self.__column_family__)
res = col_fam.get(rowkey, **kwargs)
if len(res) > 1 or len(res) == 0:
raise ModelException("get_one_by_rowkey() returned more than one "
"element or zero")
return self(rowkey, res[rowkey])
def _get_analytics_start_time_thrift(self):
try:
col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
row = col_family.get(SYSTEM_OBJECT_ANALYTICS)
return row
except Exception as e:
self._logger.error("Exception: analytics_start_time Failure %s" % e)
return None
def GetValue(pool, columnFamily, key, *args, **kwargs):
d = None
try:
col_fam = ColumnFamily(pool, columnFamily)
d = col_fam.get(key, *args, **kwargs)
except Exception,e:
#print('empty column '+key)
pass
def _get_analytics_start_time(self):
try:
col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
row = col_family.get(SYSTEM_OBJECT_ANALYTICS, columns=[SYSTEM_OBJECT_START_TIME])
except Exception as e:
self._logger.error("Exception: analytics_start_time Failure %s" % e)
return None
else:
return row[SYSTEM_OBJECT_START_TIME]
def _get_analytics_start_time_thrift(self):
try:
col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
row = col_family.get(SYSTEM_OBJECT_ANALYTICS)
return row
except Exception as e:
self._logger.error("Exception: analytics_start_time Failure %s" %
e)
return None
def test_retrieve_with_custom_composite(self):
cf_std = ColumnFamily(pool, "CustomComposite1")
cf_cust = ColumnFamily(pool, "CustomComposite1")
cf_cust.column_name_class = CompositeType(TestCustomComposite.IntDateType(), TestCustomComposite.IntString())
std_col = (20120312, "321")
cust_col = (date(2012, 3, 12), 321)
cf_std.insert("cust_insert_key_2", {std_col: "cust_insert_val_2"})
assert_equal(cf_cust.get("cust_insert_key_2"), {cust_col: "cust_insert_val_2"})
def update_thread_status(self, thread_id):
threads = ColumnFamily(self.conn, 'threads')
dt = datetime.datetime.today()
str_dt = dt.strftime('%Y-%m-%d %H:%M:%S')
ret = threads.get(str(thread_id), columns=['post_count'])
post_count = int(ret['post_count']) + 1
threads.insert(thread_id, {'post_count': str(post_count), 'update_time': str_dt})
class JobRun2:
def __init__(self):
err = ''
self.pool = ConnectionPool('jobrun', server_list=options['hosts'])
self.jl = ColumnFamily(self.pool, 'job_lookup')
self.jr = ColumnFamily(self.pool, 'job_results')
self.jd = ColumnFamily(self.pool, 'job_dashboard')
self.jf = ColumnFamily(self.pool, 'job_failures')
def closePool(self):
pass
def getLast(self,rk):
try:
jlookup = self.jl.get(rk, column_count=1)
status = self.jr.get(jlookup.values()[0],columns=['status']).values()[0]
except Exception,e:
return float(-100)
return status
def column_family_get_key(self,machine_id,keyspace_name,column_family_name,key):
"""Remove a key from column family for a given keyspace """
if (self.keyspace_contains("localhost:9160",keyspace_name,column_family_name) == False):
print "Error : Keyspace:column family could not be found."
return False
pool = ConnectionPool(keyspace = keyspace_name, server_list = keyspace.server_ips, prefill=False)
col_fam = ColumnFamily(pool, column_family_name)
print key
result = col_fam.get(key)
return result
def verify_with_thrift(self):
# No more thrift in 4.0
if self.cluster.version() >= '4':
return
pool = ConnectionPool("supcols", pool_size=1)
super_col_fam = ColumnFamily(pool, "cols")
for name in NAMES:
super_col_value = super_col_fam.get(name)
self.assertEqual(OrderedDict([(('attr', u'name'), name)]), super_col_value)
def verify_with_thrift(self):
# No more thrift in 4.0
if self.cluster.version() >= '4':
return
pool = ConnectionPool("supcols", pool_size=1)
super_col_fam = ColumnFamily(pool, "cols")
for name in NAMES:
super_col_value = super_col_fam.get(name)
self.assertEqual(OrderedDict([(('attr', u'name'), name)]),
super_col_value)
def _get_analytics_start_time(self):
try:
col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
row = col_family.get(SYSTEM_OBJECT_ANALYTICS,
columns=[SYSTEM_OBJECT_START_TIME])
except Exception as e:
self._logger.error("Exception: analytics_start_time Failure %s" %
e)
return -1
else:
return row[SYSTEM_OBJECT_START_TIME]
def getByTag(tag):
print '''-- MOVIES GROUPED BY USER FOR A GIVE TAG --'''
print '''tag: {}'''.format(tag)
con = util.getConnection()
tagCF = ColumnFamily(con, 'tag_videos_composite')
movies = tagCF.get(tag.strip().lower())
for key, val in movies.iteritems():
compositeCol = key
print '([{0}],[{1}]) => {2}'.format(compositeCol[0], compositeCol[1], val)
movieSlice = tagCF.get(tag.strip().lower(), column_start=("Kara", "The Croods:Kara"), column_finish=("Sally","Gx" ))
#movieSlice = tagCF.get(tag.strip().lower(), column_start=("Kara", ), column_finish=(("Leo Scott",False),))
print '-- SLICES --'
for key, val in movieSlice.iteritems():
compositeCol = key
print '([{0}],[{1}]) => {2}'.format(compositeCol[0], compositeCol[1], val)
con.dispose()
def test_insert_with_custom_composite(self):
cf_std = ColumnFamily(pool, 'CustomComposite1')
cf_cust = ColumnFamily(pool, 'CustomComposite1')
cf_cust.column_name_class = CompositeType(
TestCustomComposite.IntDateType(),
TestCustomComposite.IntString())
std_col = (20120311, '321')
cust_col = (date(2012, 3, 11), 321)
cf_cust.insert('cust_insert_key_1', {cust_col: 'cust_insert_val_1'})
assert_equal(cf_std.get('cust_insert_key_1'),
{std_col: 'cust_insert_val_1'})
def test_uuid_composites(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'UUIDComposite',
comparator_type=CompositeType(IntegerType(reversed=True), TimeUUIDType()),
key_validation_class=TimeUUIDType(),
default_validation_class=UTF8Type())
key, u1, u2 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1()
cf = ColumnFamily(pool, 'UUIDComposite')
cf.insert(key, {(123123, u1): 'foo'})
cf.insert(key, {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
assert_equal(cf.get(key), {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
def simple_select(self, columnfamily, *args):
slice = ['', '', self.max_rows]
key = None
if args and args[1]:
if ':' not in args[1]:
key = args[1]
for i, part in enumerate(args[1].split(':', 2)):
slice[i] = part
try:
cf = ColumnFamily(self.pool, columnfamily)
except NotFoundException:
return super(CCli, self).default(' '.join([columnfamily] + list(args)))
if key:
pt = PrettyTable()
pt.field_names = ['Key', key]
pt.align["Key"] = "l"
pt.align[key] = 'r'
for k, v in cf.get(key).items():
pt.add_row([k, (v[:self.max_data_size - 3] + '...' if self.max_data_size and len(v) > self.max_data_size else v)])
print pt.get_string(sortby='Key')
return
data = dict(cf.get_range(start=slice[0], finish=slice[1], row_count=int(slice[2])))
columns = []
for key, row in data.items():
columns.extend(row.keys())
columns = list(set(columns))
columns.sort()
pt = PrettyTable()
pt.field_names = ['Key / Column'] + columns
pt.align["Key / Column"] = "l"
for column in columns:
pt.align[column] = "r"
for key, row in data.items():
prow = [key]
for column in columns:
value = row.get(column, '---')
if len(value) > self.max_data_size:
value = value[:self.max_data_size - 3] + '...'
prow.append(value)
pt.add_row(prow)
print pt.get_string(sortby='Key / Column')
def test_validation_with_packed_names(self):
"""
Make sure that validated columns are packed correctly when the
column names themselves must be packed
"""
sys = SystemManager()
sys.create_column_family(TEST_KS, 'Validators2',
comparator_type=LongType(), default_validation_class=LongType())
sys.alter_column(TEST_KS, 'Validators2', 1, TimeUUIDType())
sys.close()
my_uuid = uuid.uuid1()
cf = ColumnFamily(pool, 'Validators2')
cf.insert('key', {0: 0})
assert_equal(cf.get('key'), {0: 0})
cf.insert('key', {1: my_uuid})
assert_equal(cf.get('key'), {0: 0, 1: my_uuid})
cf.insert('key', {0: 0, 1: my_uuid})
assert_equal(cf.get('key'), {0: 0, 1: my_uuid})
def query(pool):
cf_logs = ColumnFamily(pool, CF_LOGS)
row_key = ymd_from_epoch()
try:
cf_logs.get(row_key, column_count=0)
except NotFoundException:
# FIXME: this is extremely inefficient!
row_key = cf_logs.get_range().next()[0]
logging.info("-" * 120) # ------------------------------
logging.info("Querying for key %s", row_key)
logging.info("-" * 120) # ------------------------------
count = 20
for k, v in cf_logs.get(row_key, column_reversed=True).iteritems(): #@UnusedVariable
logging.info(v)
count -= 1
if count == 0:
break
del cf_logs
logging.info("-" * 120) # ------------------------------
cf_logs_by_app = ColumnFamily(pool, CF_LOGS_BY_APP)
row_key = EXAMPLE_APPS[0]
logging.info("Querying for key %s", row_key)
logging.info("-" * 120) # ------------------------------
count = 20
for k, v in cf_logs_by_app.get(row_key, column_reversed=True).iteritems(): #@UnusedVariable
logging.info(v)
count -= 1
if count == 0:
break
del cf_logs_by_app
logging.info("-" * 120) # ------------------------------
cf_logs_by_host = ColumnFamily(pool, CF_LOGS_BY_HOST)
row_key = EXAMPLE_HOSTS[0]
logging.info("Querying for key %s", row_key)
logging.info("-" * 120) # ------------------------------
count = 20
for k, v in cf_logs_by_host.get(row_key, column_reversed=True).iteritems(): #@UnusedVariable
logging.info(v)
count -= 1
if count == 0:
break
del cf_logs_by_host
logging.info("-" * 120) # ------------------------------
cf_logs_by_severity = ColumnFamily(pool, CF_LOGS_BY_SEVERITY)
row_key = 'WARN'
logging.info("Querying for key %s", row_key)
logging.info("-" * 120) # ------------------------------
count = 20
for k, v in cf_logs_by_severity.get(row_key, column_reversed=True).iteritems(): #@UnusedVariable
logging.info(v)
count -= 1
if count == 0:
break
del cf_logs_by_severity
def _get_analytics_ttls_thrift(self):
ret_row = {}
try:
col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
row = col_family.get(SYSTEM_OBJECT_ANALYTICS)
except Exception as e:
self._logger.error("Exception: analytics_start_time Failure %s" %
e)
ret_row[SYSTEM_OBJECT_FLOW_DATA_TTL] = AnalyticsFlowTTL
ret_row[SYSTEM_OBJECT_STATS_DATA_TTL] = AnalyticsStatisticsTTL
ret_row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] = AnalyticsConfigAuditTTL
ret_row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] = AnalyticsTTL
return (ret_row, -1)
return (row, 0)
def get_values(servlst, ks, cf, key):
#print key
try:
pool = ConnectionPool(ks, servlst)
cf_handle = ColumnFamily(pool, cf)
result = cf_handle.get(key).values()
except pycassa.NotFoundException as err:
print "[ERROR] " + key + " not found"
result = ""
except Exception as err:
print "[ERROR] " + str(err)
exit(-1)
finally:
pool.dispose()
return result
def test_add_remove_counter(self):
sys = SystemManager()
if sys._conn.version == CASSANDRA_07:
raise SkipTest("Cassandra 0.7 does not have key validators")
sys.create_column_family(TEST_KS, 'KeyLongCounter', key_validation_class=LongType(),
default_validation_class=COUNTER_COLUMN_TYPE)
sys.close()
cf_long = ColumnFamily(pool, 'KeyLongCounter')
key = 1111111111111111L
cf_long.add(key, 'col')
assert_equal(cf_long.get(key), {'col': 1})
cf_long.remove_counter(key, 'col')
time.sleep(0.1)
assert_raises(NotFoundException, cf_long.get, key)
def test_validated_columns(self):
sys = SystemManager()
sys.create_column_family(
TEST_KS,
'Validators',
)
sys.alter_column(TEST_KS, 'Validators', 'long', LongType())
sys.alter_column(TEST_KS, 'Validators', 'int', IntegerType())
sys.alter_column(TEST_KS, 'Validators', 'time', TimeUUIDType())
sys.alter_column(TEST_KS, 'Validators', 'lex', LexicalUUIDType())
sys.alter_column(TEST_KS, 'Validators', 'ascii', AsciiType())
sys.alter_column(TEST_KS, 'Validators', 'utf8', UTF8Type())
sys.alter_column(TEST_KS, 'Validators', 'bytes', BytesType())
sys.close()
cf = ColumnFamily(pool, 'Validators')
key = 'key1'
col = {'long': 1L}
cf.insert(key, col)
assert_equal(cf.get(key)['long'], 1L)
col = {'int': 1}
cf.insert(key, col)
assert_equal(cf.get(key)['int'], 1)
col = {'time': TIME1}
cf.insert(key, col)
assert_equal(cf.get(key)['time'], TIME1)
col = {'lex': uuid.UUID(bytes='aaa aaa aaa aaaa')}
cf.insert(key, col)
assert_equal(cf.get(key)['lex'], uuid.UUID(bytes='aaa aaa aaa aaaa'))
col = {'ascii': 'aaa'}
cf.insert(key, col)
assert_equal(cf.get(key)['ascii'], 'aaa')
col = {'utf8': u'a\u0020'}
cf.insert(key, col)
assert_equal(cf.get(key)['utf8'], u'a\u0020')
col = {'bytes': 'aaa'}
cf.insert(key, col)
assert_equal(cf.get(key)['bytes'], 'aaa')
cf.remove(key)
def test_validated_columns(self):
sys = SystemManager()
sys.create_column_family(
TEST_KS,
'Validators',
)
sys.alter_column(TEST_KS, 'Validators', 'long', LONG_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'int', INT_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'time', TIME_UUID_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'lex', LEXICAL_UUID_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'ascii', ASCII_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'utf8', UTF8_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'bytes', BYTES_TYPE)
sys.close()
cf = ColumnFamily(pool, 'Validators')
key = 'key1'
col = {'long': 1L}
cf.insert(key, col)
assert_equal(cf.get(key)['long'], 1L)
col = {'int': 1}
cf.insert(key, col)
assert_equal(cf.get(key)['int'], 1)
col = {'time': TIME1}
cf.insert(key, col)
assert_equal(cf.get(key)['time'], TIME1)
col = {'lex': uuid.UUID(bytes='aaa aaa aaa aaaa')}
cf.insert(key, col)
assert_equal(cf.get(key)['lex'], uuid.UUID(bytes='aaa aaa aaa aaaa'))
col = {'ascii': 'aaa'}
cf.insert(key, col)
assert_equal(cf.get(key)['ascii'], 'aaa')
col = {'utf8': u'a\u0020'}
cf.insert(key, col)
assert_equal(cf.get(key)['utf8'], u'a\u0020')
col = {'bytes': 'aaa'}
cf.insert(key, col)
assert_equal(cf.get(key)['bytes'], 'aaa')
cf.remove(key)
def test_default_validated_columns(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'DefaultValidator', default_validation_class=LongType())
sys.alter_column(TEST_KS, 'DefaultValidator', 'subcol', TimeUUIDType())
sys.close()
cf = ColumnFamily(pool, 'DefaultValidator')
key = 'key1'
col_cf = {'aaaaaa': 1L}
col_cm = {'subcol': TIME1}
col_ncf = {'aaaaaa': TIME1}
col_ncm = {'subcol': 1L}
# Both of these inserts work, as cf allows
# longs and cm for 'subcol' allows TIMEUUIDs.
cf.insert(key, col_cf)
cf.insert(key, col_cm)
assert_equal(cf.get(key), {'aaaaaa': 1L, 'subcol': TIME1})
def get(self, key, *args, **kwargs):
"""
Creates one or more instances of `cls` from the row with key `key`.
The fields that are retreived may be specified using `columns`, which
should be a list of column names.
If the column family is a super column family, a list of `cls`
instances will be returned, one for each super column. If
the `super_column` parameter is not supplied, then `columns`
specifies which super columns will be used to create instances
of `cls`. If the `super_column` parameter *is* supplied, only
one instance of `cls` will be returned; if `columns` is specified
in this case, only those attributes listed in `columns` will be fetched.
All other parameters behave the same as in :meth:`.ColumnFamily.get()`.
"""
if 'columns' not in kwargs and not self.super and not self.raw_columns:
kwargs['columns'] = self.fields
columns = ColumnFamily.get(self, key, *args, **kwargs)
if self.super:
if 'super_column' not in kwargs:
vals = self.dict_class()
for super_column, subcols in columns.iteritems():
combined = self.combine_columns(subcols)
vals[super_column] = create_instance(
self.cls,
key=key,
super_column=super_column,
**combined)
return vals
combined = self.combine_columns(columns)
return create_instance(self.cls,
key=key,
super_column=kwargs['super_column'],
**combined)
combined = self.combine_columns(columns)
return create_instance(self.cls, key=key, **combined)
# print(row)
for row in loopin:
loop_col_fam.insert(
(row['detectorid'] + ' - ' + row['starttime']), {
'detectorid': row['detectorid'],
'starttime': row['starttime'],
'volume': row['volume'],
'speed': row['speed'],
'occupancy': row['occupancy'],
'status': row['status'],
'dqflags': row['dqflags']
})
# sys.create_index('highwaydata', 'loopdata', 'detectorid', INT_TYPE)
# sys.create_index('highwaydata', 'loopdata', 'speed', INT_TYPE)
sys.create_index('highwaydata', 'loopdata', 'detectorid', UTF8_TYPE)
sys.create_index('highwaydata', 'loopdata', 'speed', UTF8_TYPE)
sys.create_index('highwaydata', 'loopdata', 'starttime', UTF8_TYPE)
print('getting info for detector & starttime 1345 - 9/15/2011 12:04:00 AM')
print(loop_col_fam.get('1345 - 2011-09-15 00:04:00-07'))
loops_end_time = time.time()
results.write("loop data took %s seconds to import" %
(loops_end_time - loops_start_time))
print("loop data took %s seconds to import" %
(loops_end_time - loops_start_time))
print('')
results.close()
sys.close()
pool.dispose()
print('all done')
def test_static_composite_get_partial_composite(self):
cf = ColumnFamily(pool, 'StaticComposite')
cf.insert('key3', {(123123, 1): 'val'})
assert_equal(cf.get('key3'), {(123123, 1): 'val'})
class Buyer(Llama):
def __init__(self, client, qname, trend=5):
super(Buyer, self).__init__(client, uuid.uuid4().hex)
self.holdings = {}
self.cash = 100000.0
self.history = {}
self.trend = trend
self.pool = ConnectionPool('example_consumer_Buyer')
self.stored_holdings = ColumnFamily(self.pool, 'Holdings')
self.quote_history = ColumnFamily(self.pool, 'Quotes')
self.stored_cash = ColumnFamily(self.pool, 'Cash')
try:
cash = self.stored_cash.get('current')
self.cash = cash['amount']
except ttypes.NotFoundException:
self.stored_cash.insert('current', {'amount': self.cash})
for symbol, columns in self.stored_holdings.get_range():
self.holdings[symbol] = (columns['number_of_shares'],
columns['price'], columns['cost'])
date_expression = create_index_expression('timestamp',
datetime.date.today(), GT)
date_clause = create_index_clause([date_expression], count=1000)
for key, columns in self.quote_history.get_range():
symbol = columns['symbol']
price = columns['price']
self.add_quote(symbol, price)
def add_quote(self, symbol, price):
if symbol not in self.history:
self.history[symbol] = [price]
else:
self.history[symbol].append(price)
if len(self.history[symbol]) >= self.trend:
price_low = min(self.history[symbol][-self.trend:])
price_max = max(self.history[symbol][-self.trend:])
price_avg = sum(self.history[symbol][-self.trend:]) / self.trend
#print "Recent history of %s is %s" % (symbol, self.history[symbol][-self.trend:])
else:
price_low, price_max, price_avg = (-1, -1, -1)
print "%s quotes until we start deciding whether to buy or sell %s" % (
self.trend - len(self.history[symbol]), symbol)
#print "Recent history of %s is %s" % (symbol, self.history[symbol])
return (price_low, price_max, price_avg)
def do_message(self, quote):
symbol, price, date, counter = quote
#print "Thinking about whether to buy or sell %s at %s" % (symbol, price)
price_low, price_max, price_avg = self.add_quote(symbol, price)
self.save_quote(symbol, price)
if price_low == -1: return
#print "Trending minimum/avg/max of %s is %s-%s-%s" % (symbol, price_low, price_avg, price_max)
#for symbol in self.holdings.keys():
# print "self.history[symbol][-1] = %s" % self.history[symbol][-1]
# print "self.holdings[symbol][0] = %s" % self.holdings[symbol][0]
# print "Value of %s is %s" % (symbol, float(self.holdings[symbol][0])*self.history[symbol][-1])
value = sum([
self.holdings[symbol][0] * self.history[symbol][-1]
for symbol in self.holdings.keys()
])
print "Net worth is %s + %s = %s" % (self.cash, value,
self.cash + value)
if symbol not in self.holdings:
if price < 1.01 * price_low:
shares_to_buy = random.choice([10, 15, 20, 25, 30])
print "I don't own any %s yet, and the price is below the trending minimum of %s so I'm buying %s shares." % (
symbol, price_low, shares_to_buy)
cost = shares_to_buy * price
print "Cost is %s, cash is %s" % (cost, self.cash)
if cost < self.cash:
self.buy_holdings(symbol, shares_to_buy, price, cost)
self.update_cash(-cost)
print "Cash is now %s" % self.cash
else:
print "Unfortunately, I don't have enough cash at this time."
else:
if price > self.holdings[symbol][1] and price > 0.99 * price_max:
print "+++++++ Price of %s is higher than my holdings, so I'm going to sell!" % symbol
sale_value = self.holdings[symbol][0] * price
print "Sale value is %s" % sale_value
print "Holdings value is %s" % self.holdings[symbol][2]
print "Total net is %s" % (sale_value -
self.holdings[symbol][2])
self.update_cash(sale_value)
print "Cash is now %s" % self.cash
self.sell_holdings(symbol)
def update_cash(self, change):
self.cash += change
cash = self.stored_cash.get('current')
cash['amount'] = self.cash
self.stored_cash.insert('current', cash)
def buy_holdings(self, symbol, shares_to_buy, price, cost):
self.holdings[symbol] = (shares_to_buy, price, cost)
stored_holding = {
'number_of_shares': shares_to_buy,
'price': price,
'cost': cost
}
self.stored_holdings.insert(symbol, stored_holding)
def sell_holdings(self, symbol):
del self.holdings[symbol]
self.stored_holdings.remove(symbol)
def save_quote(self, symbol, price):
key = str(uuid.uuid4())
self.quote_history.insert(key, {'symbol': symbol, 'price': price})
import pycassa
from pycassa.pool import ConnectionPool
from pycassa.columnfamily import ColumnFamily
import csv
pool = ConnectionPool('highwayData', ['localhost:9160'])
#change to not super column
col_fam = ColumnFamily(pool, 'stationid')
print('\n\ngetting record for station 1098 from the stations column family')
print(col_fam.get('1098'))
print('\n\n')
col_fam_detectors = ColumnFamily(pool, 'detectors')
print(
'getting record for detector 1345, 09-15-2011 from the detectors & loopdata super-column family'
)
print('record check: 1345,2011-09-15 00:00:00-07,0,,0,0,0')
print(col_fam_detectors.get(
'1345',
columns=['2011-09-15 00:00:00-07'],
))
print('\n\n')
#print(col_fam_detectors.get('1345'))
#1346,2011-09-24 21:21:20-07,7,63,11,2,0
#1348,2011-11-06 03:53:20-08,0,,0,0,0
def fetch(key):
pool = ConnectionPool(KEY_SPACE, [DB_URI])
col_fam = ColumnFamily(pool, COLUMN_FAMILY)
col_fam.insert('row_key', {'col_name': 'col_val'})
return col_fam.get(str(key))
def test_static_composite_basic(self):
cf = ColumnFamily(pool, 'StaticComposite')
colname = (127312831239123123, 1, uuid.uuid1(), uuid.uuid4(), 'foo', u'ba\u0254r', 'baz')
cf.insert('key', {colname: 'val'})
assert_equal(cf.get('key'), {colname: 'val'})
from pycassa.types import *
from pycassa.system_manager import *
from pycassa.pool import ConnectionPool
from pycassa.columnfamily import ColumnFamily
def create_ks():
# create test keyspace
sys = SystemManager()
comparator = CompositeType(LongType(), BytesType())
sys.create_column_family("testing", "testing", comparator_type=comparator)
pool = ConnectionPool('testing')
cf = ColumnFamily(pool, 'testing')
# Check the column added by the Haskell test script
# print [k for k in cf.get_range()]
# cf.insert("row2", {(125, 'oklahoma'): 'asdf'})
print cf.get('row1')
print cf.get('row2')
# should see: OrderedDict([((125, 'oklahoma'), 'asdf')])
import pycassa
import logging
from pycassa.pool import ConnectionPool
from pycassa.columnfamily import ColumnFamily
from datetime import datetime as d
__author__ = 'rahul'
logging.basicConfig(filename="example.log",level=logging.DEBUG,format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S')
pool = ConnectionPool('icepice')
col_fam = ColumnFamily(pool, 'info')
#Sample Insert operations
#col_fam.insert('row_key',{'col_name' : 'value' })
'''Sample Get operations.
Need to wrap the get operations in exception block for safe failing from
'key not found error'.
'''
try:
for i in range(0,1000):
s = d.now()
col_fam.get(i)
logging.info(str(i) +"=>"+ str((d.now() - s).microseconds))
except:
logging.info("Error" + str(sys.exc_info()[0]))
import pycassa
from pycassa.pool import ConnectionPool
from pycassa.columnfamily import ColumnFamily
pool = ConnectionPool('keyspace2', ['localhost:9160'])
col_fam = ColumnFamily(pool, 'cassandraGroup')
name = col_fam.get('Chad')
print(name)
# name = col_fam.get('Will')
# print(name)
# name = col_fam.get('Tom')
# print(name)
def execute(self):
client = db_connection.get_client()
cf = ColumnFamily(client, self.domain)
try:
#### SELECT QUERY ####
if self.op_type == CassandraQuery.OP_SELECT:
if not self.where_node:
## treat this as a simple key get query
if self.limit == 1:
result = cf.get(self.offset)
if result:
return (True, result, None)
else:
return (False, None,
DatabaseError("No " + self.domain +
"entry matching row_key: " +
self.offset))
else:
return (False, None,
DatabaseError(
"Limit for SELECT operation must be 1"))
else:
## treat this as an indexed_slices query
if self.limit == 1:
## we consider the assumption that there is only a single AND node with filtering children
index_expressions = []
for field_predicate, value in self.where_node.children:
field_predicate_list = field_predicate.split("__")
field = field_predicate_list[0]
predicate = EQ
if len(field_predicate_list) == 2:
try:
predicate = SelectManager.predicate_map[
field_predicate_list[1]]
except:
predicate = EQ
index_exp = create_index_expression(
field, value, predicate)
index_expressions.append(index_exp)
index_clause = create_index_clause(
index_expressions,
start_key=self.offset,
count=self.limit)
result = cf.get_indexed_slices(index_clause)
if result:
return (True, result, None)
else:
return (False, None,
DatabaseError("No " + self.domain +
"entry matching query: " +
self.where_node))
else:
return (False, None,
DatabaseError(
"Limit for SELECT operation must be 1"))
#### FETCH QUERY ####
elif self.op_type == CassandraQuery.OP_FETCH:
if self.limit > SelectManager.MAX_FETCH_LIMIT:
return (
False, None,
DatabaseError(
"LIMIT for FETCH operation exceeds MAX_FETCH_LIMIT(1000)"
))
if not self.where_node:
## Treat this as a key range query
key_offset = self.offset
limit = self.limit
result = {}
while True:
if limit < SelectManager.REGULAR_FETCH_LIMIT:
res = cf.get_range(key_offset, row_count=limit)
result.update(res)
break
else:
res = cf.get_range(
key_offset,
row_count=SelectManager.REGULAR_FETCH_LIMIT)
result.update(res)
if len(res) < SelectManager.REGULAR_FETCH_LIMIT:
break
else:
max_key = sorted(res.keys(), reverse=True)[0]
key_offset = max_key + 1
limit -= SelectManager.REGULAR_FETCH_LIMIT
return (True, result, None)
else:
## Treat this as a fetch query
## first create index expressions
index_expressions = []
for field_predicate, value in self.where_node.children:
field_predicate_list = field_predicate.split("__")
field = field_predicate_list[0]
predicate = EQ
if len(field_predicate_list) == 2:
try:
predicate = SelectManager.predicate_map[
field_predicate_list[1]]
except:
predicate = EQ
index_exp = create_index_expression(
field, value, predicate)
index_expressions.append(index_exp)
key_offset = self.offset
limit = self.limit
result = {}
while True:
if limit < SelectManager.REGULAR_FETCH_LIMIT:
index_clause = create_index_clause(
index_expressions,
start_key=key_offset,
count=limit)
res = cf.get_indexed_slices(index_clause)
result.update(res)
break
else:
index_clause = create_index_clause(
index_expressions,
start_key=key_offset,
count=SelectManager.REGULAR_FETCH_LIMIT)
res = cf.get_indexed_slices(index_clause)
result.update(res)
if len(res) < SelectManager.REGULAR_FETCH_LIMIT:
break
else:
max_key = sorted(res.keys(), reverse=True)[0]
key_offset = max_key + 1
limit -= SelectManager.REGULAR_FETCH_LIMIT
return (True, result, None)
except Exception, ex:
return (False, None, ex)
