class TestCustomTypes(unittest.TestCase): class IntString(types.CassandraType): @staticmethod def pack(intval): return str(intval) @staticmethod def unpack(strval): return int(strval) class IntString2(types.CassandraType): def __init__(self, *args, **kwargs): self.pack = lambda val: str(val) self.unpack = lambda val: int(val) def test_staticmethod_funcs(self): self.cf = ColumnFamily(pool, 'Standard1') self.cf.key_validation_class = TestCustomTypes.IntString() self.cf.insert(1234, {'col': 'val'}) assert_equal(self.cf.get(1234), {'col': 'val'}) def test_constructor_lambdas(self): self.cf = ColumnFamily(pool, 'Standard1') self.cf.key_validation_class = TestCustomTypes.IntString2() self.cf.insert(1234, {'col': 'val'}) assert_equal(self.cf.get(1234), {'col': 'val'})
def test_alter_column_non_bytes_type(self): sys.create_column_family(TEST_KS, 'LongCF', comparator_type=LONG_TYPE) sys.create_index(TEST_KS, 'LongCF', 3, LONG_TYPE) pool = ConnectionPool(TEST_KS) cf = ColumnFamily(pool, 'LongCF') cf.insert('key', {3: 3}) assert_equal(cf.get('key')[3], 3) sys.alter_column(TEST_KS, 'LongCF', 2, LONG_TYPE) cf = ColumnFamily(pool, 'LongCF') cf.insert('key', {2: 2}) assert_equal(cf.get('key')[2], 2)
def test_column_validators(self): validators = {'name': UTF8_TYPE, 'age': LONG_TYPE} sys.create_column_family(TEST_KS, 'ValidatedCF', column_validation_classes=validators) pool = ConnectionPool(TEST_KS) cf = ColumnFamily(pool, 'ValidatedCF') cf.insert('key', {'name': 'John', 'age': 40}) self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40}) validators = {'name': ASCII_TYPE, 'age': INT_TYPE} sys.alter_column_family(TEST_KS, 'ValidatedCF', column_validation_classes=validators) cf.load_schema() self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
def test_static_composite_slicing(self): cf = ColumnFamily(pool, 'StaticComposite') u1 = uuid.uuid1() u4 = uuid.uuid4() col0 = (0, 1, u1, u4, '', '', '') col1 = (1, 1, u1, u4, '', '', '') col2 = (1, 2, u1, u4, '', '', '') col3 = (1, 3, u1, u4, '', '', '') col4 = (2, 1, u1, u4, '', '', '') cf.insert('key2', {col0: '', col1: '', col2: '', col3: '', col4: ''}) result = cf.get('key2', column_start=((1, True),), column_finish=((1, True),)) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=(1,), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=((1, True),), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=(1, ), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=((0, False), ), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=(1, 1), column_finish=(1, 3)) assert_equal(result, {col1: '', col2: ''}) result = cf.get('key2', column_start=(1, 1), column_finish=(1, (3, True))) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=(1, (1, True)), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''})
def test_static_composite_slicing(self): cf = ColumnFamily(pool, "StaticComposite") u1 = uuid.uuid1() u4 = uuid.uuid4() col0 = (0, 1, u1, u4, "", "", "") col1 = (1, 1, u1, u4, "", "", "") col2 = (1, 2, u1, u4, "", "", "") col3 = (1, 3, u1, u4, "", "", "") col4 = (2, 1, u1, u4, "", "", "") cf.insert("key2", {col0: "", col1: "", col2: "", col3: "", col4: ""}) result = cf.get("key2", column_start=((1, True),), column_finish=((1, True),)) assert_equal(result, {col1: "", col2: "", col3: ""}) result = cf.get("key2", column_start=(1,), column_finish=((2, False),)) assert_equal(result, {col1: "", col2: "", col3: ""}) result = cf.get("key2", column_start=((1, True),), column_finish=((2, False),)) assert_equal(result, {col1: "", col2: "", col3: ""}) result = cf.get("key2", column_start=(1,), column_finish=((2, False),)) assert_equal(result, {col1: "", col2: "", col3: ""}) result = cf.get("key2", column_start=((0, False),), column_finish=((2, False),)) assert_equal(result, {col1: "", col2: "", col3: ""}) result = cf.get("key2", column_start=(1, 1), column_finish=(1, 3)) assert_equal(result, {col1: "", col2: "", col3: ""}) result = cf.get("key2", column_start=(1, 1), column_finish=(1, (3, True))) assert_equal(result, {col1: "", col2: "", col3: ""}) result = cf.get("key2", column_start=(1, (1, True)), column_finish=((2, False),)) assert_equal(result, {col1: "", col2: "", col3: ""})
def test_static_composite(cls): sys = SystemManager() have_composites = sys._conn.version != CASSANDRA_07 if not have_composites: raise SkipTest("Cassandra < 0.8 does not composite types") sys.create_column_family(TEST_KS, 'StaticComposite', comparator_type=CompositeType(LongType(), IntegerType(), TimeUUIDType(reversed=True), LexicalUUIDType(reversed=False), AsciiType(), UTF8Type(), BytesType())) cf = ColumnFamily(pool, 'StaticComposite') colname = (127312831239123123, 1, uuid.uuid1(), uuid.uuid4(), 'foo', u'ba\u0254r', 'baz') cf.insert('key', {colname: 'val'}) assert_equal(cf.get('key'), {colname: 'val'}) u1 = uuid.uuid1() u4 = uuid.uuid4() col0 = (0, 1, u1, u4, '', '', '') col1 = (1, 1, u1, u4, '', '', '') col2 = (1, 2, u1, u4, '', '', '') col3 = (1, 3, u1, u4, '', '', '') col4 = (2, 1, u1, u4, '', '', '') cf.insert('key2', {col0: '', col1: '', col2: '', col3: '', col4: ''}) result = cf.get('key2', column_start=((1, True),), column_finish=((1, True),)) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=(1,), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=((1, True),), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=(1, ), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=((0, False), ), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=(1, 1), column_finish=(1, 3)) assert_equal(result, {col1: '', col2: ''}) result = cf.get('key2', column_start=(1, 1), column_finish=(1, (3, True))) assert_equal(result, {col1: '', col2: '', col3: ''}) result = cf.get('key2', column_start=(1, (1, True)), column_finish=((2, False), )) assert_equal(result, {col1: '', col2: '', col3: ''}) sys.drop_column_family(TEST_KS, 'StaticComposite')
class Processor(Llama): def __init__(self, client, qname): super(Processor, self).__init__(client, qname) self.pool = ConnectionPool('processing_llama_Processor') self.trends = ColumnFamily(self.pool, 'Trend') def get_sleep_time(): return 60 def do_message(self, message): if not isinstance(message, tuple) or len(message) != 4: return woeid, as_of, trend_name, query = message try: trend = self.trends.get(trend_name, super_column=woeid) trend['lastseen'] = as_of trend['number_seen'] += 1 trend = { woeid: trend } self.trends.insert(trend_name, trend) except ttypes.NotFoundException: self.trends.insert(trend_name, { woeid: { 'firstseen': as_of, 'lastseen': as_of, 'number_seen': 1}}) self.trends.insert(trend_name, {'data': { 'query': query, 'tracking': "False" }}) def has_many_woeids(self, x): key, values = x return len(values) > 2 # one woeid plus general data def do_action(self): try: for trend_name, country_specifics in filter(self.has_many_woeids, self.trends.get_range()): if country_specifics['data']['tracking'] == "False": self.track_trend(trend_name, country_specifics['data']['query'], filter(lambda x: x != "data", country_specifics.keys())) except: exc, value, exctraceback = sys.exc_info() logging.error("Error in processing_llama.processor.Processor.do_action:\n" + traceback.format_exc()) traceback.print_tb(exctraceback) def track_trend(self, trend_name, query, woeids): logging.info("Tracking %s from %s" % (trend_name, woeids)) self.mark_tracking(trend_name) self.publish((trend_name, query, woeids), "trend_to_track") def mark_tracking(self, trend_name): try: trend = self.trends.get(trend_name, super_column='data') trend['tracking'] = "True" self.trends.insert(trend_name, { 'data': trend }) except ttypes.NotFoundException: pass
def get_row_key_id(domain): counter_column, counter_lock = domain_counter_map[domain] ## acquire lock before getting value of counter_lock.acquire() try: client = db_connection.get_client() cf = ColumnFamily(client, CONFIG_DOMAIN) ## get new key id id_key = cf.get(CONFIG_ROW, counter_column)[counter_column] ## increment value if not None if id_key: new_id_key = id_key + 1 cf.insert(CONFIG_ROW, {counter_column: new_id_key}, write_consistency_level=ConsistencyLevel.ALL) return id_key """ if id_key: str_id_key = str(id_key) str_id_key.zfill(MAX_PADDING_RANGE) return str_id_key else: return None """ finally: ## release lock before returning from this function counter_lock.release()
class CassandraRepository(object): def __init__(self, keyspace, column_family_name): self.pool = ConnectionPool(keyspace, cassandra_settings.NODE_POOL) self.cf = ColumnFamily(self.pool, column_family_name) self.batch = {} def add_batch(self, batch, start_time=None): """ :param batch: """ self.cf.batch_insert(batch) if start_time is not None: print 'time to insert batch: %s ms' % (int(time.time() * 1000) - start_time) def get(self, timestamp): return self.cf.get(str(timestamp)) def get_range(self, start, end): return list(self.cf.get_range(start=str(start), finish=str(end))) def close(self): self.sys.close()
def get_reverse(columnFamily, key) : "select values in a given column family" try : column = ColumnFamily(pool, columnFamily) return column.get(key,column_reversed=True) except: return {'status':0}
def _get_analytics_start_time(self): try: col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE) row = col_family.get(SYSTEM_OBJECT_ANALYTICS) except Exception as e: self._logger.error("Exception: analytics_start_time Failure %s" % e) return None # Initialize the dictionary before returning if (SYSTEM_OBJECT_START_TIME not in row): return None ret_row = {} ret_row[SYSTEM_OBJECT_START_TIME] = row[SYSTEM_OBJECT_START_TIME] if (SYSTEM_OBJECT_FLOW_START_TIME not in row): ret_row[SYSTEM_OBJECT_FLOW_START_TIME] = row[ SYSTEM_OBJECT_START_TIME] else: ret_row[SYSTEM_OBJECT_FLOW_START_TIME] = row[ SYSTEM_OBJECT_FLOW_START_TIME] if (SYSTEM_OBJECT_STAT_START_TIME not in row): ret_row[SYSTEM_OBJECT_STAT_START_TIME] = row[ SYSTEM_OBJECT_START_TIME] else: ret_row[SYSTEM_OBJECT_STAT_START_TIME] = row[ SYSTEM_OBJECT_STAT_START_TIME] if (SYSTEM_OBJECT_MSG_START_TIME not in row): ret_row[SYSTEM_OBJECT_MSG_START_TIME] = row[ SYSTEM_OBJECT_START_TIME] else: ret_row[SYSTEM_OBJECT_MSG_START_TIME] = row[ SYSTEM_OBJECT_MSG_START_TIME] return ret_row
def _get_analytics_start_time(self): try: col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE) row = col_family.get(SYSTEM_OBJECT_ANALYTICS) except Exception as e: self._logger.error("Exception: analytics_start_time Failure %s" % e) return None # Initialize the dictionary before returning if (SYSTEM_OBJECT_START_TIME not in row): return None ret_row = {} ret_row[SYSTEM_OBJECT_START_TIME] = row[SYSTEM_OBJECT_START_TIME] if (SYSTEM_OBJECT_FLOW_START_TIME not in row): ret_row[SYSTEM_OBJECT_FLOW_START_TIME] = row[SYSTEM_OBJECT_START_TIME] else: ret_row[SYSTEM_OBJECT_FLOW_START_TIME] = row[SYSTEM_OBJECT_FLOW_START_TIME] if (SYSTEM_OBJECT_STAT_START_TIME not in row): ret_row[SYSTEM_OBJECT_STAT_START_TIME] = row[SYSTEM_OBJECT_START_TIME] else: ret_row[SYSTEM_OBJECT_STAT_START_TIME] = row[SYSTEM_OBJECT_STAT_START_TIME] if (SYSTEM_OBJECT_MSG_START_TIME not in row): ret_row[SYSTEM_OBJECT_MSG_START_TIME] = row[SYSTEM_OBJECT_START_TIME] else: ret_row[SYSTEM_OBJECT_MSG_START_TIME] = row[SYSTEM_OBJECT_MSG_START_TIME] return ret_row
def _get_analytics_ttls(self): ret_row = {} try: col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE) row = col_family.get(SYSTEM_OBJECT_ANALYTICS) except Exception as e: self._logger.error("Exception: analytics_start_time Failure %s" % e) ret_row[SYSTEM_OBJECT_FLOW_DATA_TTL] = AnalyticsFlowTTL ret_row[SYSTEM_OBJECT_STATS_DATA_TTL] = AnalyticsStatisticsTTL ret_row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] = AnalyticsConfigAuditTTL ret_row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] = AnalyticsTTL return ret_row if (SYSTEM_OBJECT_FLOW_DATA_TTL not in row): ret_row[SYSTEM_OBJECT_FLOW_DATA_TTL] = AnalyticsFlowTTL else: ret_row[SYSTEM_OBJECT_FLOW_DATA_TTL] = row[SYSTEM_OBJECT_FLOW_DATA_TTL] if (SYSTEM_OBJECT_STATS_DATA_TTL not in row): ret_row[SYSTEM_OBJECT_STATS_DATA_TTL] = AnalyticsStatisticsTTL else: ret_row[SYSTEM_OBJECT_STATS_DATA_TTL] = row[SYSTEM_OBJECT_STATS_DATA_TTL] if (SYSTEM_OBJECT_CONFIG_AUDIT_TTL not in row): ret_row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] = AnalyticsConfigAuditTTL else: ret_row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] = row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] if (SYSTEM_OBJECT_GLOBAL_DATA_TTL not in row): ret_row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] = AnalyticsTTL else: ret_row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] = row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] return ret_row
def build_from_id(self, build_id): """Obtain information about a build from its ID.""" cf = ColumnFamily(self.pool, 'builds') try: return cf.get(build_id) except NotFoundException: return None
def get_builder(self, builder_id): """Obtain info about a builder from its ID.""" cf = ColumnFamily(self.pool, 'builders') try: return cf.get(builder_id) except NotFoundException: return None
def getByTag(tag): con = util.getConnection() vidCF = ColumnFamily(con, 'videos') tagCF = ColumnFamily(con, 'tag_videos') movies = tagCF.get(tag.strip().lower()) for key, val in movies.iteritems(): vidId = key.split('_')[1] movieDetail = vidCF.get(vidId) print ''' {{ user: {0}, movie: {1}, tags: {2} }}'''.format(movieDetail['user_name'], movieDetail['title'], movieDetail['tags_csv']) con.dispose()
def show(query, cass): my_column_family = ColumnFamily(cass, 'my_column_family') try: result = my_column_family.get(1167864277) return result except NotFoundException, nfe: return HTTPError(404, 'Entity not found.')
def test_single_component_composite(self): sys = SystemManager() sys.create_column_family(TEST_KS, 'SingleComposite', comparator_type=CompositeType(IntegerType())) cf = ColumnFamily(pool, 'SingleComposite') cf.insert('key', {(123456,): 'val'}) assert_equal(cf.get('key'), {(123456,): 'val'})
def get_one_by_rowkey(self, rowkey, **kwargs): """Get the object by the rowkey. Supports pycassa method `get` kwargs.""" col_fam = ColumnFamily(self.pool, self.__column_family__) res = col_fam.get(rowkey, **kwargs) if len(res) > 1 or len(res) == 0: raise ModelException("get_one_by_rowkey() returned more than one " "element or zero") return self(rowkey, res[rowkey])
def _get_analytics_start_time_thrift(self): try: col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE) row = col_family.get(SYSTEM_OBJECT_ANALYTICS) return row except Exception as e: self._logger.error("Exception: analytics_start_time Failure %s" % e) return None
def GetValue(pool, columnFamily, key, *args, **kwargs): d = None try: col_fam = ColumnFamily(pool, columnFamily) d = col_fam.get(key, *args, **kwargs) except Exception,e: #print('empty column '+key) pass
def _get_analytics_start_time(self): try: col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE) row = col_family.get(SYSTEM_OBJECT_ANALYTICS, columns=[SYSTEM_OBJECT_START_TIME]) except Exception as e: self._logger.error("Exception: analytics_start_time Failure %s" % e) return None else: return row[SYSTEM_OBJECT_START_TIME]
def test_retrieve_with_custom_composite(self): cf_std = ColumnFamily(pool, "CustomComposite1") cf_cust = ColumnFamily(pool, "CustomComposite1") cf_cust.column_name_class = CompositeType(TestCustomComposite.IntDateType(), TestCustomComposite.IntString()) std_col = (20120312, "321") cust_col = (date(2012, 3, 12), 321) cf_std.insert("cust_insert_key_2", {std_col: "cust_insert_val_2"}) assert_equal(cf_cust.get("cust_insert_key_2"), {cust_col: "cust_insert_val_2"})
def update_thread_status(self, thread_id): threads = ColumnFamily(self.conn, 'threads') dt = datetime.datetime.today() str_dt = dt.strftime('%Y-%m-%d %H:%M:%S') ret = threads.get(str(thread_id), columns=['post_count']) post_count = int(ret['post_count']) + 1 threads.insert(thread_id, {'post_count': str(post_count), 'update_time': str_dt})
class JobRun2: def __init__(self): err = '' self.pool = ConnectionPool('jobrun', server_list=options['hosts']) self.jl = ColumnFamily(self.pool, 'job_lookup') self.jr = ColumnFamily(self.pool, 'job_results') self.jd = ColumnFamily(self.pool, 'job_dashboard') self.jf = ColumnFamily(self.pool, 'job_failures') def closePool(self): pass def getLast(self,rk): try: jlookup = self.jl.get(rk, column_count=1) status = self.jr.get(jlookup.values()[0],columns=['status']).values()[0] except Exception,e: return float(-100) return status
def column_family_get_key(self,machine_id,keyspace_name,column_family_name,key): """Remove a key from column family for a given keyspace """ if (self.keyspace_contains("localhost:9160",keyspace_name,column_family_name) == False): print "Error : Keyspace:column family could not be found." return False pool = ConnectionPool(keyspace = keyspace_name, server_list = keyspace.server_ips, prefill=False) col_fam = ColumnFamily(pool, column_family_name) print key result = col_fam.get(key) return result
def verify_with_thrift(self): # No more thrift in 4.0 if self.cluster.version() >= '4': return pool = ConnectionPool("supcols", pool_size=1) super_col_fam = ColumnFamily(pool, "cols") for name in NAMES: super_col_value = super_col_fam.get(name) self.assertEqual(OrderedDict([(('attr', u'name'), name)]), super_col_value)
def _get_analytics_start_time(self): try: col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE) row = col_family.get(SYSTEM_OBJECT_ANALYTICS, columns=[SYSTEM_OBJECT_START_TIME]) except Exception as e: self._logger.error("Exception: analytics_start_time Failure %s" % e) return -1 else: return row[SYSTEM_OBJECT_START_TIME]
def getByTag(tag): print '''-- MOVIES GROUPED BY USER FOR A GIVE TAG --''' print '''tag: {}'''.format(tag) con = util.getConnection() tagCF = ColumnFamily(con, 'tag_videos_composite') movies = tagCF.get(tag.strip().lower()) for key, val in movies.iteritems(): compositeCol = key print '([{0}],[{1}]) => {2}'.format(compositeCol[0], compositeCol[1], val) movieSlice = tagCF.get(tag.strip().lower(), column_start=("Kara", "The Croods:Kara"), column_finish=("Sally","Gx" )) #movieSlice = tagCF.get(tag.strip().lower(), column_start=("Kara", ), column_finish=(("Leo Scott",False),)) print '-- SLICES --' for key, val in movieSlice.iteritems(): compositeCol = key print '([{0}],[{1}]) => {2}'.format(compositeCol[0], compositeCol[1], val) con.dispose()
def test_insert_with_custom_composite(self): cf_std = ColumnFamily(pool, 'CustomComposite1') cf_cust = ColumnFamily(pool, 'CustomComposite1') cf_cust.column_name_class = CompositeType( TestCustomComposite.IntDateType(), TestCustomComposite.IntString()) std_col = (20120311, '321') cust_col = (date(2012, 3, 11), 321) cf_cust.insert('cust_insert_key_1', {cust_col: 'cust_insert_val_1'}) assert_equal(cf_std.get('cust_insert_key_1'), {std_col: 'cust_insert_val_1'})
def test_uuid_composites(self): sys = SystemManager() sys.create_column_family(TEST_KS, 'UUIDComposite', comparator_type=CompositeType(IntegerType(reversed=True), TimeUUIDType()), key_validation_class=TimeUUIDType(), default_validation_class=UTF8Type()) key, u1, u2 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1() cf = ColumnFamily(pool, 'UUIDComposite') cf.insert(key, {(123123, u1): 'foo'}) cf.insert(key, {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'}) assert_equal(cf.get(key), {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
def simple_select(self, columnfamily, *args): slice = ['', '', self.max_rows] key = None if args and args[1]: if ':' not in args[1]: key = args[1] for i, part in enumerate(args[1].split(':', 2)): slice[i] = part try: cf = ColumnFamily(self.pool, columnfamily) except NotFoundException: return super(CCli, self).default(' '.join([columnfamily] + list(args))) if key: pt = PrettyTable() pt.field_names = ['Key', key] pt.align["Key"] = "l" pt.align[key] = 'r' for k, v in cf.get(key).items(): pt.add_row([k, (v[:self.max_data_size - 3] + '...' if self.max_data_size and len(v) > self.max_data_size else v)]) print pt.get_string(sortby='Key') return data = dict(cf.get_range(start=slice[0], finish=slice[1], row_count=int(slice[2]))) columns = [] for key, row in data.items(): columns.extend(row.keys()) columns = list(set(columns)) columns.sort() pt = PrettyTable() pt.field_names = ['Key / Column'] + columns pt.align["Key / Column"] = "l" for column in columns: pt.align[column] = "r" for key, row in data.items(): prow = [key] for column in columns: value = row.get(column, '---') if len(value) > self.max_data_size: value = value[:self.max_data_size - 3] + '...' prow.append(value) pt.add_row(prow) print pt.get_string(sortby='Key / Column')
def test_validation_with_packed_names(self): """ Make sure that validated columns are packed correctly when the column names themselves must be packed """ sys = SystemManager() sys.create_column_family(TEST_KS, 'Validators2', comparator_type=LongType(), default_validation_class=LongType()) sys.alter_column(TEST_KS, 'Validators2', 1, TimeUUIDType()) sys.close() my_uuid = uuid.uuid1() cf = ColumnFamily(pool, 'Validators2') cf.insert('key', {0: 0}) assert_equal(cf.get('key'), {0: 0}) cf.insert('key', {1: my_uuid}) assert_equal(cf.get('key'), {0: 0, 1: my_uuid}) cf.insert('key', {0: 0, 1: my_uuid}) assert_equal(cf.get('key'), {0: 0, 1: my_uuid})
def query(pool): cf_logs = ColumnFamily(pool, CF_LOGS) row_key = ymd_from_epoch() try: cf_logs.get(row_key, column_count=0) except NotFoundException: # FIXME: this is extremely inefficient! row_key = cf_logs.get_range().next()[0] logging.info("-" * 120) # ------------------------------ logging.info("Querying for key %s", row_key) logging.info("-" * 120) # ------------------------------ count = 20 for k, v in cf_logs.get(row_key, column_reversed=True).iteritems(): #@UnusedVariable logging.info(v) count -= 1 if count == 0: break del cf_logs logging.info("-" * 120) # ------------------------------ cf_logs_by_app = ColumnFamily(pool, CF_LOGS_BY_APP) row_key = EXAMPLE_APPS[0] logging.info("Querying for key %s", row_key) logging.info("-" * 120) # ------------------------------ count = 20 for k, v in cf_logs_by_app.get(row_key, column_reversed=True).iteritems(): #@UnusedVariable logging.info(v) count -= 1 if count == 0: break del cf_logs_by_app logging.info("-" * 120) # ------------------------------ cf_logs_by_host = ColumnFamily(pool, CF_LOGS_BY_HOST) row_key = EXAMPLE_HOSTS[0] logging.info("Querying for key %s", row_key) logging.info("-" * 120) # ------------------------------ count = 20 for k, v in cf_logs_by_host.get(row_key, column_reversed=True).iteritems(): #@UnusedVariable logging.info(v) count -= 1 if count == 0: break del cf_logs_by_host logging.info("-" * 120) # ------------------------------ cf_logs_by_severity = ColumnFamily(pool, CF_LOGS_BY_SEVERITY) row_key = 'WARN' logging.info("Querying for key %s", row_key) logging.info("-" * 120) # ------------------------------ count = 20 for k, v in cf_logs_by_severity.get(row_key, column_reversed=True).iteritems(): #@UnusedVariable logging.info(v) count -= 1 if count == 0: break del cf_logs_by_severity
def _get_analytics_ttls_thrift(self): ret_row = {} try: col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE) row = col_family.get(SYSTEM_OBJECT_ANALYTICS) except Exception as e: self._logger.error("Exception: analytics_start_time Failure %s" % e) ret_row[SYSTEM_OBJECT_FLOW_DATA_TTL] = AnalyticsFlowTTL ret_row[SYSTEM_OBJECT_STATS_DATA_TTL] = AnalyticsStatisticsTTL ret_row[SYSTEM_OBJECT_CONFIG_AUDIT_TTL] = AnalyticsConfigAuditTTL ret_row[SYSTEM_OBJECT_GLOBAL_DATA_TTL] = AnalyticsTTL return (ret_row, -1) return (row, 0)
def get_values(servlst, ks, cf, key): #print key try: pool = ConnectionPool(ks, servlst) cf_handle = ColumnFamily(pool, cf) result = cf_handle.get(key).values() except pycassa.NotFoundException as err: print "[ERROR] " + key + " not found" result = "" except Exception as err: print "[ERROR] " + str(err) exit(-1) finally: pool.dispose() return result
def test_add_remove_counter(self): sys = SystemManager() if sys._conn.version == CASSANDRA_07: raise SkipTest("Cassandra 0.7 does not have key validators") sys.create_column_family(TEST_KS, 'KeyLongCounter', key_validation_class=LongType(), default_validation_class=COUNTER_COLUMN_TYPE) sys.close() cf_long = ColumnFamily(pool, 'KeyLongCounter') key = 1111111111111111L cf_long.add(key, 'col') assert_equal(cf_long.get(key), {'col': 1}) cf_long.remove_counter(key, 'col') time.sleep(0.1) assert_raises(NotFoundException, cf_long.get, key)
def test_validated_columns(self): sys = SystemManager() sys.create_column_family( TEST_KS, 'Validators', ) sys.alter_column(TEST_KS, 'Validators', 'long', LongType()) sys.alter_column(TEST_KS, 'Validators', 'int', IntegerType()) sys.alter_column(TEST_KS, 'Validators', 'time', TimeUUIDType()) sys.alter_column(TEST_KS, 'Validators', 'lex', LexicalUUIDType()) sys.alter_column(TEST_KS, 'Validators', 'ascii', AsciiType()) sys.alter_column(TEST_KS, 'Validators', 'utf8', UTF8Type()) sys.alter_column(TEST_KS, 'Validators', 'bytes', BytesType()) sys.close() cf = ColumnFamily(pool, 'Validators') key = 'key1' col = {'long': 1L} cf.insert(key, col) assert_equal(cf.get(key)['long'], 1L) col = {'int': 1} cf.insert(key, col) assert_equal(cf.get(key)['int'], 1) col = {'time': TIME1} cf.insert(key, col) assert_equal(cf.get(key)['time'], TIME1) col = {'lex': uuid.UUID(bytes='aaa aaa aaa aaaa')} cf.insert(key, col) assert_equal(cf.get(key)['lex'], uuid.UUID(bytes='aaa aaa aaa aaaa')) col = {'ascii': 'aaa'} cf.insert(key, col) assert_equal(cf.get(key)['ascii'], 'aaa') col = {'utf8': u'a\u0020'} cf.insert(key, col) assert_equal(cf.get(key)['utf8'], u'a\u0020') col = {'bytes': 'aaa'} cf.insert(key, col) assert_equal(cf.get(key)['bytes'], 'aaa') cf.remove(key)
def test_validated_columns(self): sys = SystemManager() sys.create_column_family( TEST_KS, 'Validators', ) sys.alter_column(TEST_KS, 'Validators', 'long', LONG_TYPE) sys.alter_column(TEST_KS, 'Validators', 'int', INT_TYPE) sys.alter_column(TEST_KS, 'Validators', 'time', TIME_UUID_TYPE) sys.alter_column(TEST_KS, 'Validators', 'lex', LEXICAL_UUID_TYPE) sys.alter_column(TEST_KS, 'Validators', 'ascii', ASCII_TYPE) sys.alter_column(TEST_KS, 'Validators', 'utf8', UTF8_TYPE) sys.alter_column(TEST_KS, 'Validators', 'bytes', BYTES_TYPE) sys.close() cf = ColumnFamily(pool, 'Validators') key = 'key1' col = {'long': 1L} cf.insert(key, col) assert_equal(cf.get(key)['long'], 1L) col = {'int': 1} cf.insert(key, col) assert_equal(cf.get(key)['int'], 1) col = {'time': TIME1} cf.insert(key, col) assert_equal(cf.get(key)['time'], TIME1) col = {'lex': uuid.UUID(bytes='aaa aaa aaa aaaa')} cf.insert(key, col) assert_equal(cf.get(key)['lex'], uuid.UUID(bytes='aaa aaa aaa aaaa')) col = {'ascii': 'aaa'} cf.insert(key, col) assert_equal(cf.get(key)['ascii'], 'aaa') col = {'utf8': u'a\u0020'} cf.insert(key, col) assert_equal(cf.get(key)['utf8'], u'a\u0020') col = {'bytes': 'aaa'} cf.insert(key, col) assert_equal(cf.get(key)['bytes'], 'aaa') cf.remove(key)
def test_default_validated_columns(self): sys = SystemManager() sys.create_column_family(TEST_KS, 'DefaultValidator', default_validation_class=LongType()) sys.alter_column(TEST_KS, 'DefaultValidator', 'subcol', TimeUUIDType()) sys.close() cf = ColumnFamily(pool, 'DefaultValidator') key = 'key1' col_cf = {'aaaaaa': 1L} col_cm = {'subcol': TIME1} col_ncf = {'aaaaaa': TIME1} col_ncm = {'subcol': 1L} # Both of these inserts work, as cf allows # longs and cm for 'subcol' allows TIMEUUIDs. cf.insert(key, col_cf) cf.insert(key, col_cm) assert_equal(cf.get(key), {'aaaaaa': 1L, 'subcol': TIME1})
def get(self, key, *args, **kwargs): """ Creates one or more instances of `cls` from the row with key `key`. The fields that are retreived may be specified using `columns`, which should be a list of column names. If the column family is a super column family, a list of `cls` instances will be returned, one for each super column. If the `super_column` parameter is not supplied, then `columns` specifies which super columns will be used to create instances of `cls`. If the `super_column` parameter *is* supplied, only one instance of `cls` will be returned; if `columns` is specified in this case, only those attributes listed in `columns` will be fetched. All other parameters behave the same as in :meth:`.ColumnFamily.get()`. """ if 'columns' not in kwargs and not self.super and not self.raw_columns: kwargs['columns'] = self.fields columns = ColumnFamily.get(self, key, *args, **kwargs) if self.super: if 'super_column' not in kwargs: vals = self.dict_class() for super_column, subcols in columns.iteritems(): combined = self.combine_columns(subcols) vals[super_column] = create_instance( self.cls, key=key, super_column=super_column, **combined) return vals combined = self.combine_columns(columns) return create_instance(self.cls, key=key, super_column=kwargs['super_column'], **combined) combined = self.combine_columns(columns) return create_instance(self.cls, key=key, **combined)
# print(row) for row in loopin: loop_col_fam.insert( (row['detectorid'] + ' - ' + row['starttime']), { 'detectorid': row['detectorid'], 'starttime': row['starttime'], 'volume': row['volume'], 'speed': row['speed'], 'occupancy': row['occupancy'], 'status': row['status'], 'dqflags': row['dqflags'] }) # sys.create_index('highwaydata', 'loopdata', 'detectorid', INT_TYPE) # sys.create_index('highwaydata', 'loopdata', 'speed', INT_TYPE) sys.create_index('highwaydata', 'loopdata', 'detectorid', UTF8_TYPE) sys.create_index('highwaydata', 'loopdata', 'speed', UTF8_TYPE) sys.create_index('highwaydata', 'loopdata', 'starttime', UTF8_TYPE) print('getting info for detector & starttime 1345 - 9/15/2011 12:04:00 AM') print(loop_col_fam.get('1345 - 2011-09-15 00:04:00-07')) loops_end_time = time.time() results.write("loop data took %s seconds to import" % (loops_end_time - loops_start_time)) print("loop data took %s seconds to import" % (loops_end_time - loops_start_time)) print('') results.close() sys.close() pool.dispose() print('all done')
def test_static_composite_get_partial_composite(self): cf = ColumnFamily(pool, 'StaticComposite') cf.insert('key3', {(123123, 1): 'val'}) assert_equal(cf.get('key3'), {(123123, 1): 'val'})
class Buyer(Llama): def __init__(self, client, qname, trend=5): super(Buyer, self).__init__(client, uuid.uuid4().hex) self.holdings = {} self.cash = 100000.0 self.history = {} self.trend = trend self.pool = ConnectionPool('example_consumer_Buyer') self.stored_holdings = ColumnFamily(self.pool, 'Holdings') self.quote_history = ColumnFamily(self.pool, 'Quotes') self.stored_cash = ColumnFamily(self.pool, 'Cash') try: cash = self.stored_cash.get('current') self.cash = cash['amount'] except ttypes.NotFoundException: self.stored_cash.insert('current', {'amount': self.cash}) for symbol, columns in self.stored_holdings.get_range(): self.holdings[symbol] = (columns['number_of_shares'], columns['price'], columns['cost']) date_expression = create_index_expression('timestamp', datetime.date.today(), GT) date_clause = create_index_clause([date_expression], count=1000) for key, columns in self.quote_history.get_range(): symbol = columns['symbol'] price = columns['price'] self.add_quote(symbol, price) def add_quote(self, symbol, price): if symbol not in self.history: self.history[symbol] = [price] else: self.history[symbol].append(price) if len(self.history[symbol]) >= self.trend: price_low = min(self.history[symbol][-self.trend:]) price_max = max(self.history[symbol][-self.trend:]) price_avg = sum(self.history[symbol][-self.trend:]) / self.trend #print "Recent history of %s is %s" % (symbol, self.history[symbol][-self.trend:]) else: price_low, price_max, price_avg = (-1, -1, -1) print "%s quotes until we start deciding whether to buy or sell %s" % ( self.trend - len(self.history[symbol]), symbol) #print "Recent history of %s is %s" % (symbol, self.history[symbol]) return (price_low, price_max, price_avg) def do_message(self, quote): symbol, price, date, counter = quote #print "Thinking about whether to buy or sell %s at %s" % (symbol, price) price_low, price_max, price_avg = self.add_quote(symbol, price) self.save_quote(symbol, price) if price_low == -1: return #print "Trending minimum/avg/max of %s is %s-%s-%s" % (symbol, price_low, price_avg, price_max) #for symbol in self.holdings.keys(): # print "self.history[symbol][-1] = %s" % self.history[symbol][-1] # print "self.holdings[symbol][0] = %s" % self.holdings[symbol][0] # print "Value of %s is %s" % (symbol, float(self.holdings[symbol][0])*self.history[symbol][-1]) value = sum([ self.holdings[symbol][0] * self.history[symbol][-1] for symbol in self.holdings.keys() ]) print "Net worth is %s + %s = %s" % (self.cash, value, self.cash + value) if symbol not in self.holdings: if price < 1.01 * price_low: shares_to_buy = random.choice([10, 15, 20, 25, 30]) print "I don't own any %s yet, and the price is below the trending minimum of %s so I'm buying %s shares." % ( symbol, price_low, shares_to_buy) cost = shares_to_buy * price print "Cost is %s, cash is %s" % (cost, self.cash) if cost < self.cash: self.buy_holdings(symbol, shares_to_buy, price, cost) self.update_cash(-cost) print "Cash is now %s" % self.cash else: print "Unfortunately, I don't have enough cash at this time." else: if price > self.holdings[symbol][1] and price > 0.99 * price_max: print "+++++++ Price of %s is higher than my holdings, so I'm going to sell!" % symbol sale_value = self.holdings[symbol][0] * price print "Sale value is %s" % sale_value print "Holdings value is %s" % self.holdings[symbol][2] print "Total net is %s" % (sale_value - self.holdings[symbol][2]) self.update_cash(sale_value) print "Cash is now %s" % self.cash self.sell_holdings(symbol) def update_cash(self, change): self.cash += change cash = self.stored_cash.get('current') cash['amount'] = self.cash self.stored_cash.insert('current', cash) def buy_holdings(self, symbol, shares_to_buy, price, cost): self.holdings[symbol] = (shares_to_buy, price, cost) stored_holding = { 'number_of_shares': shares_to_buy, 'price': price, 'cost': cost } self.stored_holdings.insert(symbol, stored_holding) def sell_holdings(self, symbol): del self.holdings[symbol] self.stored_holdings.remove(symbol) def save_quote(self, symbol, price): key = str(uuid.uuid4()) self.quote_history.insert(key, {'symbol': symbol, 'price': price})
import pycassa from pycassa.pool import ConnectionPool from pycassa.columnfamily import ColumnFamily import csv pool = ConnectionPool('highwayData', ['localhost:9160']) #change to not super column col_fam = ColumnFamily(pool, 'stationid') print('\n\ngetting record for station 1098 from the stations column family') print(col_fam.get('1098')) print('\n\n') col_fam_detectors = ColumnFamily(pool, 'detectors') print( 'getting record for detector 1345, 09-15-2011 from the detectors & loopdata super-column family' ) print('record check: 1345,2011-09-15 00:00:00-07,0,,0,0,0') print(col_fam_detectors.get( '1345', columns=['2011-09-15 00:00:00-07'], )) print('\n\n') #print(col_fam_detectors.get('1345')) #1346,2011-09-24 21:21:20-07,7,63,11,2,0 #1348,2011-11-06 03:53:20-08,0,,0,0,0
def fetch(key): pool = ConnectionPool(KEY_SPACE, [DB_URI]) col_fam = ColumnFamily(pool, COLUMN_FAMILY) col_fam.insert('row_key', {'col_name': 'col_val'}) return col_fam.get(str(key))
def test_static_composite_basic(self): cf = ColumnFamily(pool, 'StaticComposite') colname = (127312831239123123, 1, uuid.uuid1(), uuid.uuid4(), 'foo', u'ba\u0254r', 'baz') cf.insert('key', {colname: 'val'}) assert_equal(cf.get('key'), {colname: 'val'})
from pycassa.types import * from pycassa.system_manager import * from pycassa.pool import ConnectionPool from pycassa.columnfamily import ColumnFamily def create_ks(): # create test keyspace sys = SystemManager() comparator = CompositeType(LongType(), BytesType()) sys.create_column_family("testing", "testing", comparator_type=comparator) pool = ConnectionPool('testing') cf = ColumnFamily(pool, 'testing') # Check the column added by the Haskell test script # print [k for k in cf.get_range()] # cf.insert("row2", {(125, 'oklahoma'): 'asdf'}) print cf.get('row1') print cf.get('row2') # should see: OrderedDict([((125, 'oklahoma'), 'asdf')])
import pycassa import logging from pycassa.pool import ConnectionPool from pycassa.columnfamily import ColumnFamily from datetime import datetime as d __author__ = 'rahul' logging.basicConfig(filename="example.log",level=logging.DEBUG,format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S') pool = ConnectionPool('icepice') col_fam = ColumnFamily(pool, 'info') #Sample Insert operations #col_fam.insert('row_key',{'col_name' : 'value' }) '''Sample Get operations. Need to wrap the get operations in exception block for safe failing from 'key not found error'. ''' try: for i in range(0,1000): s = d.now() col_fam.get(i) logging.info(str(i) +"=>"+ str((d.now() - s).microseconds)) except: logging.info("Error" + str(sys.exc_info()[0]))
import pycassa from pycassa.pool import ConnectionPool from pycassa.columnfamily import ColumnFamily pool = ConnectionPool('keyspace2', ['localhost:9160']) col_fam = ColumnFamily(pool, 'cassandraGroup') name = col_fam.get('Chad') print(name) # name = col_fam.get('Will') # print(name) # name = col_fam.get('Tom') # print(name)
def execute(self): client = db_connection.get_client() cf = ColumnFamily(client, self.domain) try: #### SELECT QUERY #### if self.op_type == CassandraQuery.OP_SELECT: if not self.where_node: ## treat this as a simple key get query if self.limit == 1: result = cf.get(self.offset) if result: return (True, result, None) else: return (False, None, DatabaseError("No " + self.domain + "entry matching row_key: " + self.offset)) else: return (False, None, DatabaseError( "Limit for SELECT operation must be 1")) else: ## treat this as an indexed_slices query if self.limit == 1: ## we consider the assumption that there is only a single AND node with filtering children index_expressions = [] for field_predicate, value in self.where_node.children: field_predicate_list = field_predicate.split("__") field = field_predicate_list[0] predicate = EQ if len(field_predicate_list) == 2: try: predicate = SelectManager.predicate_map[ field_predicate_list[1]] except: predicate = EQ index_exp = create_index_expression( field, value, predicate) index_expressions.append(index_exp) index_clause = create_index_clause( index_expressions, start_key=self.offset, count=self.limit) result = cf.get_indexed_slices(index_clause) if result: return (True, result, None) else: return (False, None, DatabaseError("No " + self.domain + "entry matching query: " + self.where_node)) else: return (False, None, DatabaseError( "Limit for SELECT operation must be 1")) #### FETCH QUERY #### elif self.op_type == CassandraQuery.OP_FETCH: if self.limit > SelectManager.MAX_FETCH_LIMIT: return ( False, None, DatabaseError( "LIMIT for FETCH operation exceeds MAX_FETCH_LIMIT(1000)" )) if not self.where_node: ## Treat this as a key range query key_offset = self.offset limit = self.limit result = {} while True: if limit < SelectManager.REGULAR_FETCH_LIMIT: res = cf.get_range(key_offset, row_count=limit) result.update(res) break else: res = cf.get_range( key_offset, row_count=SelectManager.REGULAR_FETCH_LIMIT) result.update(res) if len(res) < SelectManager.REGULAR_FETCH_LIMIT: break else: max_key = sorted(res.keys(), reverse=True)[0] key_offset = max_key + 1 limit -= SelectManager.REGULAR_FETCH_LIMIT return (True, result, None) else: ## Treat this as a fetch query ## first create index expressions index_expressions = [] for field_predicate, value in self.where_node.children: field_predicate_list = field_predicate.split("__") field = field_predicate_list[0] predicate = EQ if len(field_predicate_list) == 2: try: predicate = SelectManager.predicate_map[ field_predicate_list[1]] except: predicate = EQ index_exp = create_index_expression( field, value, predicate) index_expressions.append(index_exp) key_offset = self.offset limit = self.limit result = {} while True: if limit < SelectManager.REGULAR_FETCH_LIMIT: index_clause = create_index_clause( index_expressions, start_key=key_offset, count=limit) res = cf.get_indexed_slices(index_clause) result.update(res) break else: index_clause = create_index_clause( index_expressions, start_key=key_offset, count=SelectManager.REGULAR_FETCH_LIMIT) res = cf.get_indexed_slices(index_clause) result.update(res) if len(res) < SelectManager.REGULAR_FETCH_LIMIT: break else: max_key = sorted(res.keys(), reverse=True)[0] key_offset = max_key + 1 limit -= SelectManager.REGULAR_FETCH_LIMIT return (True, result, None) except Exception, ex: return (False, None, ex)