def insert(self, events): if not events: return batch_stmt = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=ConsistencyLevel.QUORUM) shard_idx = {} for _id, event in events: shard_time = round_down(event[TIMESTAMP_FIELD], self.width) shard = shard_idx.get(shard_time, random.randint(0, self.shards - 1)) # Insert to index. try: self.index_cache.get((shard_time, shard)) except KeyError: batch_stmt.add(BoundStatement(self.namespace.INDEX_INSERT_STMT, routing_key=self.stream, consistency_level=ConsistencyLevel.QUORUM) .bind((self.stream, shard_time, self.width, shard))) self.index_cache.set((shard_time, shard), None) # Insert to stream. shard_key = StreamShard.get_key(self.stream, shard_time, shard) batch_stmt.add(BoundStatement(self.namespace.INSERT_STMT, routing_key=shard_key, consistency_level=ConsistencyLevel.QUORUM) .bind((shard_key, _id, marshal.dumps(event)))) shard_idx[shard_time] = (shard + 1) % self.shards # Round robin. self.session.execute(batch_stmt)
def PrepareStatements(cls, keyspace=None): if keyspace: if cls.SetKeyspace(keyspace) == dictionary.UNSUCCESSFUL_OPERATION: return dictionary.UNSUCCESSFUL_OPERATION cmd_select = """ SELECT * FROM {0} WHERE year = ? AND month = ? AND id = ?; """.format(cls.offers_table) cmd_select_all = """ SELECT * FROM {0} """.format(cls.offers_table) try: cls.select_stmt = cls.session.prepare(cmd_select) prepared_stmt = cls.session.prepare(cmd_select_all) cls.select_all_stmt = BoundStatement(prepared_stmt, fetch_size=10) except InvalidRequest: print("Tabla no configurada") raise return dictionary.UNSUCCESSFUL_OPERATION return dictionary.SUCCESSFUL_OPERATION
def test_invalid_argument_type(self): keyspace = 'keyspace1' column_family = 'cf1' column_metadata = [(keyspace, column_family, 'foo1', Int32Type), (keyspace, column_family, 'foo2', Int32Type)] prepared_statement = PreparedStatement(column_metadata=column_metadata, query_id=None, routing_key_indexes=[], query=None, keyspace=keyspace) bound_statement = BoundStatement(prepared_statement=prepared_statement) values = ['nonint', 1] try: bound_statement.bind(values) except TypeError as e: self.assertIn('foo1', str(e)) self.assertIn('Int32Type', str(e)) self.assertIn('str', str(e)) else: self.fail('Passed invalid type but exception was not thrown') values = [1, ['1', '2']] try: bound_statement.bind(values) except TypeError as e: self.assertIn('foo2', str(e)) self.assertIn('Int32Type', str(e)) self.assertIn('list', str(e)) else: self.fail('Passed invalid type but exception was not thrown')
def start_fetching_events_async(self, start_id, end_id): if self.descending: select_stmt = self.namespace.SELECT_DESC_STMT else: select_stmt = self.namespace.SELECT_ASC_STMT select_stmt = BoundStatement(select_stmt, fetch_size=self.read_size or 5000, routing_key=self.key, consistency_level=ConsistencyLevel.ONE) self._events_future = self.session.execute_async( select_stmt.bind((self.key, start_id, end_id, self.limit)))
def ids_iterator(self, start_id, end_id): select_ids_stmt = BoundStatement(self.namespace.SELECT_ID_STMT, fetch_size=100000, # 100k ids at a time. routing_key=self.key, consistency_level=ConsistencyLevel.ONE) ids = self.session.execute( select_ids_stmt.bind((self.key, start_id, end_id, self.limit))) for _id in ids: try: yield _id[0] except GeneratorExit: break
def setUpClass(cls): cls.prepared = PreparedStatement(column_metadata=[ ColumnMetadata('keyspace', 'cf', 'rk0', Int32Type), ColumnMetadata('keyspace', 'cf', 'rk1', Int32Type), ColumnMetadata('keyspace', 'cf', 'ck0', Int32Type), ColumnMetadata('keyspace', 'cf', 'v0', Int32Type) ], query_id=None, routing_key_indexes=[1, 0], query=None, keyspace='keyspace', protocol_version=cls.protocol_version) cls.bound = BoundStatement(prepared_statement=cls.prepared)
def test_inherit_fetch_size(self): keyspace = 'keyspace1' column_family = 'cf1' column_metadata = [(keyspace, column_family, 'foo1', Int32Type), (keyspace, column_family, 'foo2', Int32Type)] prepared_statement = PreparedStatement(column_metadata=column_metadata, query_id=None, routing_key_indexes=[], query=None, keyspace=keyspace, protocol_version=2, fetch_size=1234) bound_statement = BoundStatement(prepared_statement=prepared_statement) self.assertEqual(1234, bound_statement.fetch_size)
def delete_from_shard(shard): batch_stmt = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=ConsistencyLevel.QUORUM) num_deleted = 0 shard = StreamShard(self.namespace, self.stream, shard['start_time'], shard['width'], shard['shard'], False, MAX_LIMIT, read_size=self.read_size) for _id in shard.ids_iterator(start_id, end_id): if _id == start_id: continue num_deleted += 1 batch_stmt.add(BoundStatement(self.namespace.DELETE_STMT, routing_key=shard.key, consistency_level=ConsistencyLevel.QUORUM) .bind((shard.key, _id))) self.session.execute(batch_stmt) return num_deleted
def get_overlapping_shards(self, start_time, end_time): index_scan_stmt = BoundStatement(self.namespace.INDEX_SCAN_STMT) potential_shards = self.session.execute( index_scan_stmt.bind((self.stream, max(start_time - Stream.MAX_WIDTH, 0), end_time)) ) shards = defaultdict(lambda: defaultdict(int)) for (shard_time, width, shard) in potential_shards: if shard_time + width < start_time: # end_time < shard start_time? continue shards[shard_time][shard] = max(shards[shard_time][shard], width) for shard_time, _ in shards.iteritems(): for shard, width in _.iteritems(): yield {'start_time': shard_time, 'width': width, 'shard': shard}
def test_invalid_argument_type(self): keyspace = 'keyspace1' column_family = 'cf1' column_metadata = [(keyspace, column_family, 'foo1', Int32Type), (keyspace, column_family, 'foo2', Int32Type)] prepared_statement = PreparedStatement(column_metadata=column_metadata, query_id=None, routing_key_indexes=[], query=None, keyspace=keyspace) bound_statement = BoundStatement(prepared_statement=prepared_statement) values = ['nonint', 1] try: bound_statement.bind(values) except InvalidParameterTypeError as e: self.assertEqual(e.col_name, 'foo1') self.assertEqual(e.expected_type, Int32Type) self.assertEqual(e.actual_type, str) else: self.fail('Passed invalid type but exception was not thrown') try: bound_statement.bind(values) except TypeError as e: self.assertEqual(e.col_name, 'foo1') self.assertEqual(e.expected_type, Int32Type) self.assertEqual(e.actual_type, str) else: self.fail('Passed invalid type but exception was not thrown') values = [1, ['1', '2']] try: bound_statement.bind(values) except InvalidParameterTypeError as e: self.assertEqual(e.col_name, 'foo2') self.assertEqual(e.expected_type, Int32Type) self.assertEqual(e.actual_type, list) else: self.fail('Passed invalid type but exception was not thrown')
def get_pressure_readings_for_journey(self, spacecraft_name, journey_id, page_size=25, page_state=None): stmt = BoundStatement(self.select_prep_stmt, fetch_size=int(page_size)).bind({ 'spacecraft_name': spacecraft_name, 'journey_id': uuid_from_string(journey_id) }) # note we must decode the page_state using the same codec as the encoding on the controller side # this is necessary because the page state is a binary blob and can not be automatically handled by json result = self._session.execute( stmt, paging_state=codecs.decode(page_state, 'hex') if page_state else None) return result
def _create_statement(self, query, use_prepared=False, routing_key=None, **kwargs): if use_prepared: # prepared statements should only be generated once on the # server and then reused. If we have not generated # a prepared, go ahead and prepare it if query not in self.PREPARED_STATEMENT_CACHE: self.PREPARED_STATEMENT_CACHE[query] = ( self.session.prepare(query) ) prepared = self.PREPARED_STATEMENT_CACHE[query] return BoundStatement(prepared, **kwargs) if routing_key: routing_key = self._pack_routing_key(routing_key) elif self.MISSING_ROUTING_KEY_WARNING: _logger.warning( "The following query is missing a routing key: %s", query ) return self._create_simple_statement(query, routing_key=routing_key, **kwargs)
def get_sites(self): stmt = BoundStatement(self.select) result = self._session.execute(stmt) return result