def insert(self, events):
if not events:
return
batch_stmt = BatchStatement(batch_type=BatchType.UNLOGGED,
consistency_level=ConsistencyLevel.QUORUM)
shard_idx = {}
for _id, event in events:
shard_time = round_down(event[TIMESTAMP_FIELD], self.width)
shard = shard_idx.get(shard_time,
random.randint(0, self.shards - 1))
# Insert to index.
try:
self.index_cache.get((shard_time, shard))
except KeyError:
batch_stmt.add(BoundStatement(self.namespace.INDEX_INSERT_STMT,
routing_key=self.stream,
consistency_level=ConsistencyLevel.QUORUM)
.bind((self.stream, shard_time, self.width, shard)))
self.index_cache.set((shard_time, shard), None)
# Insert to stream.
shard_key = StreamShard.get_key(self.stream, shard_time, shard)
batch_stmt.add(BoundStatement(self.namespace.INSERT_STMT,
routing_key=shard_key,
consistency_level=ConsistencyLevel.QUORUM)
.bind((shard_key,
_id,
marshal.dumps(event))))
shard_idx[shard_time] = (shard + 1) % self.shards # Round robin.
self.session.execute(batch_stmt)
def start_fetching_events_async(self, start_id, end_id):
if self.descending:
select_stmt = self.namespace.SELECT_DESC_STMT
else:
select_stmt = self.namespace.SELECT_ASC_STMT
select_stmt = BoundStatement(select_stmt,
fetch_size=self.read_size or 5000,
routing_key=self.key,
consistency_level=ConsistencyLevel.ONE)
self._events_future = self.session.execute_async(
select_stmt.bind((self.key, start_id, end_id, self.limit)))
def ids_iterator(self, start_id, end_id):
select_ids_stmt = BoundStatement(self.namespace.SELECT_ID_STMT,
fetch_size=100000, # 100k ids at a time.
routing_key=self.key,
consistency_level=ConsistencyLevel.ONE)
ids = self.session.execute(
select_ids_stmt.bind((self.key, start_id, end_id, self.limit)))
for _id in ids:
try:
yield _id[0]
except GeneratorExit:
break
def PrepareStatements(cls, keyspace=None):
if keyspace:
if cls.SetKeyspace(keyspace) == dictionary.UNSUCCESSFUL_OPERATION:
return dictionary.UNSUCCESSFUL_OPERATION
cmd_select = """
SELECT * FROM {0} WHERE
year = ? AND
month = ? AND
id = ?;
""".format(cls.offers_table)
cmd_select_all = """
SELECT * FROM {0}
""".format(cls.offers_table)
try:
cls.select_stmt = cls.session.prepare(cmd_select)
prepared_stmt = cls.session.prepare(cmd_select_all)
cls.select_all_stmt = BoundStatement(prepared_stmt, fetch_size=10)
except InvalidRequest:
print("Tabla no configurada")
raise
return dictionary.UNSUCCESSFUL_OPERATION
return dictionary.SUCCESSFUL_OPERATION
def get_overlapping_shards(self, start_time, end_time):
index_scan_stmt = BoundStatement(self.namespace.INDEX_SCAN_STMT)
potential_shards = self.session.execute(
index_scan_stmt.bind((self.stream,
max(start_time - Stream.MAX_WIDTH, 0),
end_time))
)
shards = defaultdict(lambda: defaultdict(int))
for (shard_time, width, shard) in potential_shards:
if shard_time + width < start_time:
# end_time < shard start_time?
continue
shards[shard_time][shard] = max(shards[shard_time][shard], width)
for shard_time, _ in shards.iteritems():
for shard, width in _.iteritems():
yield {'start_time': shard_time,
'width': width,
'shard': shard}
def test_invalid_argument_type(self):
keyspace = 'keyspace1'
column_family = 'cf1'
column_metadata = [(keyspace, column_family, 'foo1', Int32Type),
(keyspace, column_family, 'foo2', Int32Type)]
prepared_statement = PreparedStatement(column_metadata=column_metadata,
query_id=None,
routing_key_indexes=[],
query=None,
keyspace=keyspace)
bound_statement = BoundStatement(prepared_statement=prepared_statement)
values = ['nonint', 1]
try:
bound_statement.bind(values)
except TypeError as e:
self.assertIn('foo1', str(e))
self.assertIn('Int32Type', str(e))
self.assertIn('str', str(e))
else:
self.fail('Passed invalid type but exception was not thrown')
values = [1, ['1', '2']]
try:
bound_statement.bind(values)
except TypeError as e:
self.assertIn('foo2', str(e))
self.assertIn('Int32Type', str(e))
self.assertIn('list', str(e))
else:
self.fail('Passed invalid type but exception was not thrown')
def setUpClass(cls):
cls.prepared = PreparedStatement(column_metadata=[
ColumnMetadata('keyspace', 'cf', 'rk0', Int32Type),
ColumnMetadata('keyspace', 'cf', 'rk1', Int32Type),
ColumnMetadata('keyspace', 'cf', 'ck0', Int32Type),
ColumnMetadata('keyspace', 'cf', 'v0', Int32Type)
],
query_id=None,
routing_key_indexes=[1, 0],
query=None,
keyspace='keyspace',
protocol_version=cls.protocol_version)
cls.bound = BoundStatement(prepared_statement=cls.prepared)
def test_invalid_argument_type(self):
keyspace = 'keyspace1'
column_family = 'cf1'
column_metadata = [
(keyspace, column_family, 'foo1', Int32Type),
(keyspace, column_family, 'foo2', Int32Type)
]
prepared_statement = PreparedStatement(column_metadata=column_metadata,
query_id=None,
routing_key_indexes=[],
query=None,
keyspace=keyspace,
protocol_version=2)
bound_statement = BoundStatement(prepared_statement=prepared_statement)
values = ['nonint', 1]
try:
bound_statement.bind(values)
except TypeError as e:
self.assertIn('foo1', str(e))
self.assertIn('Int32Type', str(e))
self.assertIn('str', str(e))
else:
self.fail('Passed invalid type but exception was not thrown')
values = [1, ['1', '2']]
try:
bound_statement.bind(values)
except TypeError as e:
self.assertIn('foo2', str(e))
self.assertIn('Int32Type', str(e))
self.assertIn('list', str(e))
else:
self.fail('Passed invalid type but exception was not thrown')
def test_inherit_fetch_size(self):
keyspace = 'keyspace1'
column_family = 'cf1'
column_metadata = [(keyspace, column_family, 'foo1', Int32Type),
(keyspace, column_family, 'foo2', Int32Type)]
prepared_statement = PreparedStatement(column_metadata=column_metadata,
query_id=None,
routing_key_indexes=[],
query=None,
keyspace=keyspace,
protocol_version=2,
fetch_size=1234)
bound_statement = BoundStatement(prepared_statement=prepared_statement)
self.assertEqual(1234, bound_statement.fetch_size)
def test_invalid_argument_type(self):
keyspace = 'keyspace1'
column_family = 'cf1'
column_metadata = [(keyspace, column_family, 'foo1', Int32Type),
(keyspace, column_family, 'foo2', Int32Type)]
prepared_statement = PreparedStatement(column_metadata=column_metadata,
query_id=None,
routing_key_indexes=[],
query=None,
keyspace=keyspace)
bound_statement = BoundStatement(prepared_statement=prepared_statement)
values = ['nonint', 1]
try:
bound_statement.bind(values)
except InvalidParameterTypeError as e:
self.assertEqual(e.col_name, 'foo1')
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, str)
else:
self.fail('Passed invalid type but exception was not thrown')
try:
bound_statement.bind(values)
except TypeError as e:
self.assertEqual(e.col_name, 'foo1')
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, str)
else:
self.fail('Passed invalid type but exception was not thrown')
values = [1, ['1', '2']]
try:
bound_statement.bind(values)
except InvalidParameterTypeError as e:
self.assertEqual(e.col_name, 'foo2')
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, list)
else:
self.fail('Passed invalid type but exception was not thrown')
def delete_from_shard(shard):
batch_stmt = BatchStatement(batch_type=BatchType.UNLOGGED,
consistency_level=ConsistencyLevel.QUORUM)
num_deleted = 0
shard = StreamShard(self.namespace, self.stream,
shard['start_time'], shard['width'],
shard['shard'], False,
MAX_LIMIT, read_size=self.read_size)
for _id in shard.ids_iterator(start_id, end_id):
if _id == start_id:
continue
num_deleted += 1
batch_stmt.add(BoundStatement(self.namespace.DELETE_STMT,
routing_key=shard.key,
consistency_level=ConsistencyLevel.QUORUM)
.bind((shard.key, _id)))
self.session.execute(batch_stmt)
return num_deleted
def test_invalid_argument_type(self):
keyspace = 'keyspace1'
column_family = 'cf1'
column_metadata = [
(keyspace, column_family, 'foo1', Int32Type),
(keyspace, column_family, 'foo2', Int32Type)
]
prepared_statement = PreparedStatement(column_metadata=column_metadata,
query_id=None,
routing_key_indexes=[],
query=None,
keyspace=keyspace)
bound_statement = BoundStatement(prepared_statement=prepared_statement)
values = ['nonint', 1]
try:
bound_statement.bind(values)
except InvalidParameterTypeError as e:
self.assertEqual(e.col_name, 'foo1')
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, str)
else:
self.fail('Passed invalid type but exception was not thrown')
try:
bound_statement.bind(values)
except TypeError as e:
self.assertEqual(e.col_name, 'foo1')
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, str)
else:
self.fail('Passed invalid type but exception was not thrown')
values = [1, ['1', '2']]
try:
bound_statement.bind(values)
except InvalidParameterTypeError as e:
self.assertEqual(e.col_name, 'foo2')
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, list)
else:
self.fail('Passed invalid type but exception was not thrown')
def get_pressure_readings_for_journey(self,
spacecraft_name,
journey_id,
page_size=25,
page_state=None):
stmt = BoundStatement(self.select_prep_stmt,
fetch_size=int(page_size)).bind({
'spacecraft_name':
spacecraft_name,
'journey_id':
uuid_from_string(journey_id)
})
# note we must decode the page_state using the same codec as the encoding on the controller side
# this is necessary because the page state is a binary blob and can not be automatically handled by json
result = self._session.execute(
stmt,
paging_state=codecs.decode(page_state, 'hex')
if page_state else None)
return result
def test_invalid_argument_type(self):
keyspace = "keyspace1"
column_family = "cf1"
column_metadata = [(keyspace, column_family, "foo1", Int32Type), (keyspace, column_family, "foo2", Int32Type)]
prepared_statement = PreparedStatement(
column_metadata=column_metadata, query_id=None, routing_key_indexes=[], query=None, keyspace=keyspace
)
bound_statement = BoundStatement(prepared_statement=prepared_statement)
values = ["nonint", 1]
try:
bound_statement.bind(values)
except InvalidParameterTypeError as e:
self.assertEqual(e.col_name, "foo1")
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, str)
else:
self.fail("Passed invalid type but exception was not thrown")
try:
bound_statement.bind(values)
except TypeError as e:
self.assertEqual(e.col_name, "foo1")
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, str)
else:
self.fail("Passed invalid type but exception was not thrown")
values = [1, ["1", "2"]]
try:
bound_statement.bind(values)
except InvalidParameterTypeError as e:
self.assertEqual(e.col_name, "foo2")
self.assertEqual(e.expected_type, Int32Type)
self.assertEqual(e.actual_type, list)
else:
self.fail("Passed invalid type but exception was not thrown")
def _create_statement(self, query, use_prepared=False, routing_key=None,
**kwargs):
if use_prepared:
# prepared statements should only be generated once on the
# server and then reused. If we have not generated
# a prepared, go ahead and prepare it
if query not in self.PREPARED_STATEMENT_CACHE:
self.PREPARED_STATEMENT_CACHE[query] = (
self.session.prepare(query)
)
prepared = self.PREPARED_STATEMENT_CACHE[query]
return BoundStatement(prepared, **kwargs)
if routing_key:
routing_key = self._pack_routing_key(routing_key)
elif self.MISSING_ROUTING_KEY_WARNING:
_logger.warning(
"The following query is missing a routing key: %s",
query
)
return self._create_simple_statement(query, routing_key=routing_key,
**kwargs)
def get_sites(self):
stmt = BoundStatement(self.select)
result = self._session.execute(stmt)
return result
