def WriteData(self, data):
# return if no data is received to write
if (0 == len(data)):
WriteBehindLog(
"Warning: in connector received empty batch to write")
return
if not self.session:
self.session = self.connection.Connect()
# in case of exactly once get last id written
shardId = None
try:
# Get the entry corresponding to shard id in exactly once table
if ((self.exactlyOnceTableName is not None)
and isinstance(self.connection, RedisConnection)):
shardId = 'shard-%s' % hashtag()
res = self.session.execute_command('HGET',
self.exactlyOnceTableName,
shardId)
if res is not None:
self.exactlyOnceLastId = str(res)
else:
self.shouldCompareId = False
except Exception as e:
self.session = None #Next time we will connect to database
self.exactlyOnceLastId = None
self.shouldCompareId = True if self.exactlyOnceTableName is not None else False
msg = "Exception occur while getting shard id for exactly once. Exception : {}".format(
e)
WriteBehindLog(msg)
raise Exception(msg) from None
pipe = self.session.pipeline()
try:
# data is list of dictionaries
# iterate over one by one and extract a dictionary and process it
lastStreamId = None
for d in data:
d_val = d['value'] # get value of key 'value' from dictionary
lastStreamId = d.pop(
'id', None) # pop the stream id out of the record
if self.shouldCompareId and CompareIds(self.exactlyOnceLastId,
lastStreamId) >= 0:
WriteBehindLog(
'Skip {} as it was already written to the backend'.
format(lastStreamId))
continue
self.shouldCompareId = False
# check operation permission, what gets replicated
op = d_val.pop(OP_KEY,
None) # pop the operation key out of the record
if op not in self.supportedOperations:
msg = 'Got unknown operation'
raise Exception(msg) from None
pk = d_val.pop(SIMPLE_HASH_BACKEND_PK)
newKey = '{}:{}'.format(self.new_prefix, pk)
if op != OPERATION_UPDATE_REPLICATE:
# pipeline key to delete
pipe.delete(newKey)
else:
# pipeline key and field-value mapping to set
pipe.hset(newKey, mapping=d_val)
# make entry for exactly once. In case of Redis cluster exception will be raised already
if ((self.exactlyOnceTableName is not None)
and isinstance(self.connection, RedisConnection)):
l_exact_once_val = {shardId: lastStreamId}
pipe.hset(self.exactlyOnceTableName,
mapping=l_exact_once_val)
#execute pipeline ommands
pipe.execute()
except Exception as e:
self.session.close()
self.session = None # next time we will reconnect to the database
self.exactlyOnceLastId = None
self.shouldCompareId = True if self.exactlyOnceTableName is not None else False
msg = "Got exception when writing to DB, Exception : {}".format(e)
WriteBehindLog(msg)
raise Exception(msg) from None
def WriteData(self, data):
if len(data) == 0:
WriteBehindLog('Warning, got an empty batch')
return
query = None
try:
if not self.conn:
from sqlalchemy.sql import text
self.sqlText = text
self.conn = self.connection.Connect()
if self.exactlyOnceTableName is not None:
shardId = 'shard-%s' % hashtag()
result = self.conn.execute(
self.sqlText(
'select val from %s where id=:id' %
self.exactlyOnceTableName, {'id': shardId}))
res = result.first()
if res is not None:
self.exactlyOnceLastId = str(res['val'])
else:
self.shouldCompareId = False
except Exception as e:
self.conn = None # next time we will reconnect to the database
self.exactlyOnceLastId = None
self.shouldCompareId = True if self.exactlyOnceTableName is not None else False
msg = 'Failed connecting to SQL database, error="%s"' % str(e)
WriteBehindLog(msg)
raise Exception(msg) from None
idsToAck = []
trans = self.conn.begin()
try:
batch = []
isAddBatch = True if data[0]['value'][
OP_KEY] == OPERATION_UPDATE_REPLICATE else False
query = self.addQuery if isAddBatch else self.delQuery
lastStreamId = None
for d in data:
x = d['value']
lastStreamId = d.pop(
'id', None
) # pop the stream id out of the record, we do not need it.
if self.shouldCompareId and CompareIds(self.exactlyOnceLastId,
lastStreamId) >= 0:
WriteBehindLog(
'Skip %s as it was already writen to the backend' %
lastStreamId)
continue
op = x.pop(OP_KEY, None)
if op not in self.supportedOperations:
msg = 'Got unknown operation'
WriteBehindLog(msg)
raise Exception(msg) from None
self.shouldCompareId = False
if op != OPERATION_UPDATE_REPLICATE: # we have only key name, it means that the key was deleted
if isAddBatch:
self.conn.execute(self.sqlText(query), batch)
batch = []
isAddBatch = False
query = self.delQuery
batch.append(x)
else:
if not isAddBatch:
self.conn.execute(self.sqlText(query), batch)
batch = []
isAddBatch = True
query = self.addQuery
batch.append(x)
if len(batch) > 0:
self.conn.execute(self.sqlText(query), batch)
if self.exactlyOnceTableName is not None:
self.conn.execute(self.sqlText(self.exactlyOnceQuery), {
'id': shardId,
'val': lastStreamId
})
trans.commit()
except Exception as e:
try:
trans.rollback()
except Exception as e:
WriteBehindLog('Failed rollback transaction')
self.conn = None # next time we will reconnect to the database
self.exactlyOnceLastId = None
self.shouldCompareId = True if self.exactlyOnceTableName is not None else False
msg = 'Got exception when writing to DB, query="%s", error="%s".' % (
(query if query else 'None'), str(e))
WriteBehindLog(msg)
raise Exception(msg) from None
def WriteData(self, data):
if len(data) == 0:
WriteBehindLog('Warning, got an empty batch')
return
query = None
try:
if not self.session:
self.session = self.connection.Connect()
if self.exactlyOnceTableName is not None:
shardId = 'shard-%s' % hashtag()
result = self.session.execute('select val from %s where id=?' % self.exactlyOnceTableName, shardId)
res = result.first()
if res is not None:
self.exactlyOnceLastId = str(res['val'])
else:
self.shouldCompareId = False
except Exception as e:
self.session = None # next time we will reconnect to the database
self.exactlyOnceLastId = None
self.shouldCompareId = True if self.exactlyOnceTableName is not None else False
msg = 'Failed connecting to Cassandra database, error="%s"' % str(e)
WriteBehindLog(msg)
raise Exception(msg) from None
idsToAck = []
try:
from cassandra.cluster import BatchStatement
batch = BatchStatement()
isAddBatch = True if data[0]['value'][OP_KEY] == OPERATION_UPDATE_REPLICATE else False
query = self.addQuery if isAddBatch else self.delQuery
stmt = self.session.prepare(query)
lastStreamId = None
for d in data:
x = d['value']
lastStreamId = d.pop('id', None) # pop the stream id out of the record, we do not need it
if self.shouldCompareId and CompareIds(self.exactlyOnceLastId, lastStreamId) >= 0:
WriteBehindLog('Skip %s as it was already writen to the backend' % lastStreamId)
continue
op = x.pop(OP_KEY, None)
if op not in self.supportedOperations:
msg = 'Got unknown operation'
WriteBehindLog(msg)
raise Exception(msg) from None
self.shouldCompareId = False
if op != OPERATION_UPDATE_REPLICATE:
if isAddBatch:
self.session.execute(batch)
batch = BatchStatement()
isAddBatch = False
query = self.delQuery
else:
if not isAddBatch:
self.session.execute(batch)
batch = BatchStatement()
isAddBatch = True
query = self.addQuery
stmt = self.session.prepare(query)
batch.add(stmt.bind(x))
if len(batch) > 0:
self.session.execute(batch)
if self.exactlyOnceTableName is not None:
stmt = self.session.prepare(self.exactlyOnceQuery)
self.session.execute(stmt, {'id':shardId, 'val':lastStreamId})
except Exception as e:
self.session = None # next time we will reconnect to the database
self.exactlyOnceLastId = None
self.shouldCompareId = True if self.exactlyOnceTableName is not None else False
msg = 'Got exception when writing to DB, query="%s", error="%s".' % ((query if query else 'None'), str(e))
WriteBehindLog(msg)
raise Exception(msg) from None
def GetStreamName(tableName):
return '_%s-stream-%s-{%s}' % (tableName, uid, hashtag())