def test_publish_for_dtr_multithread(self):
self.assertEqual(True, utilsRedis.deleteElements(tableName))
getConfig(config_file_path, DEBUG, reload=True)\
.set('General', 'numberofthreads', 2)\
.set('Dtr', 'active', 'yes')\
.set('Dtr', 'inputchannel', 'test_channel')
channel = utilsRedis.getRedisChannelListener('test_channel')
self.assertEqual(True, bool(channel))
rta_dltest_db_redis = RTA_DLTEST_DB('redis-basic', config_file_path)
sleep(0.3)
countMessages = 0
for message in channel.listen():
countMessages += 1
break
rta_dltest_db_redis.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
for message in channel.listen():
countMessages += 1
break
s1 = rta_dltest_db_redis.waitAndClose()
self.assertEqual(1, s1[0])
for message in channel.listen():
countMessages += 1
break
self.assertEqual(3, countMessages)
def test_asynchronous_single_thread_streaming_insert(self):
self.assertEqual(True, utilsMySql.deleteElements(tableName))
self.assertEqual(True, utilsRedis.deleteElements(tableName))
getConfig(config_file_path, DEBUG, reload=True)\
.set('General', 'numberofthreads', 1)\
.set('General', 'batchsize', 1)
rta_dltest_db_mysql = RTA_DLTEST_DB('mysql', config_file_path)
rta_dltest_db_redis = RTA_DLTEST_DB('redis-basic', config_file_path)
self.assertEqual(rta_dltest_db_mysql.multithreading, True)
self.assertEqual(rta_dltest_db_redis.multithreading, True)
rta_dltest_db_mysql.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
rta_dltest_db_redis.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
sleep(0.3)
self.assertEqual(1, utilsMySql.countElements(tableName))
self.assertEqual(1, utilsRedis.countElements(tableName))
s1 = rta_dltest_db_mysql.waitAndClose()
s2 = rta_dltest_db_redis.waitAndClose()
self.assertEqual(1, s1[0])
self.assertEqual(1, s2[0])
def test_connection(self):
getConfig(config_file_path, DEBUG,
reload=True).set('General', 'numberofthreads', 0)
rta_evt1_db_mysql = RTA_EVT1_DB('mysql', config_file_path)
rta_evt1_db_redis = RTA_EVT1_DB('redis-basic', config_file_path)
self.assertEqual(True, rta_evt1_db_mysql.dbConnector.testConnection())
self.assertEqual(True, rta_evt1_db_redis.dbConnector.testConnection())
def test_batch_closing_connection_before_finish(self):
self.assertEqual(True, utils.deleteElements(tableName))
getConfig(config_file_path, DEBUG,
reload=True).set('General', 'batchsize', 2)
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(True, mysqlConn.connect())
self.assertEqual(
MySqlDBConnector.SUCCESS,
mysqlConn.insertData(tableName, {
'a': 1,
'b': 2,
'c': 3,
'd': 4
}))
self.assertEqual(1, mysqlConn.conn.in_transaction)
self.assertEqual(1, mysqlConn.commandsSent)
self.assertEqual(0, utils.countElements(tableName))
self.assertEqual(True,
mysqlConn.close()) # <--- committing on closing..
self.assertEqual(1, utils.countElements(tableName))
def test_connection_with_wrong_username(self):
getConfig(config_file_path, DEBUG, reload=True)\
.set('MySql', 'password', 'asdasd')\
.set('MySql', 'username', 'gioacchino')
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(False, mysqlConn.connect())
def test_set_synchronous_mode(self):
getConfig(config_file_path, DEBUG,
reload=True).set('General', 'numberofthreads', 0)
rta_dltest_db_mysql = RTA_DLTEST_DB('mysql', config_file_path)
rta_dltest_db_redis = RTA_DLTEST_DB('redis-basic', config_file_path)
self.assertEqual(rta_dltest_db_mysql.multithreading, False)
self.assertEqual(rta_dltest_db_redis.multithreading, False)
def test_batch_insert(self):
self.assertEqual(True, utils.deleteElements(tableName))
getConfig(config_file_path, DEBUG, reload=True)
getConfig(config_file_path, DEBUG).set('General', 'batchsize', 2)
redisConn = RedisDBConnectorBASIC(config_file_path)
self.assertEqual(True, redisConn.connect())
self.assertEqual(
True, redisConn.insertData(tableName, {
'a': 4,
'b': 2,
'c': 3
}))
self.assertEqual(0, utils.countElements(tableName))
self.assertEqual(
True, redisConn.insertData(tableName, {
'a': 2,
'b': 2,
'c': 3
}))
self.assertEqual(2, utils.countElements(tableName))
self.assertEqual(True, redisConn.close())
def test_connection_success(self):
getConfig(config_file_path, DEBUG, reload=True)
redisConn = RedisDBConnectorBASIC(config_file_path)
self.assertEqual(True, redisConn.connect())
self.assertEqual(True, redisConn.testConnection())
self.assertEqual(True, redisConn.close())
def test_streaming_insert(self):
self.assertEqual(True, utils.deleteElements(tableName))
getConfig(config_file_path, DEBUG, reload=True)
redisConn = RedisDBConnectorBASIC(config_file_path)
self.assertEqual(True, redisConn.connect())
self.assertEqual(
True, redisConn.insertData(tableName, {
'a': 4,
'b': 2,
'c': 3
}))
self.assertEqual(
True, redisConn.insertData(tableName, {
'a': 2,
'b': 2,
'c': 3
}))
self.assertEqual(
True, redisConn.insertData(tableName, {
'a': 9,
'b': 2,
'c': 3
}))
self.assertEqual(True, redisConn.close())
self.assertEqual(3, utils.countElements(tableName))
def test_connectionion_success(self):
getConfig(config_file_path, DEBUG, reload=True)
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(True, mysqlConn.connect())
self.assertEqual(True, mysqlConn.testConnection())
self.assertEqual(True, mysqlConn.close())
def test_asynchronous_multi_thread_batch_insert_closing_connection_before_finish(
self):
self.assertEqual(True, utilsMySql.deleteElements(tableName))
self.assertEqual(True, utilsRedis.deleteElements(tableName))
getConfig(config_file_path, DEBUG, reload=True)\
.set('General', 'numberofthreads', 2)\
.set('General', 'batchsize', 3)
rta_dltest_db_mysql = RTA_DLTEST_DB('mysql', config_file_path)
rta_dltest_db_redis = RTA_DLTEST_DB('redis-basic', config_file_path)
self.assertEqual(rta_dltest_db_mysql.multithreading, True)
self.assertEqual(rta_dltest_db_redis.multithreading, True)
for i in range(2):
rta_dltest_db_mysql.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
rta_dltest_db_redis.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
sleep(0.5)
self.assertEqual(0, utilsMySql.countElements(tableName))
self.assertEqual(0, utilsRedis.countElements(tableName))
s1 = rta_dltest_db_mysql.waitAndClose()
s2 = rta_dltest_db_redis.waitAndClose()
self.assertEqual(2, s1[0])
self.assertEqual(2, s2[0])
self.assertEqual(2, utilsMySql.countElements(tableName))
self.assertEqual(2, utilsRedis.countElements(tableName))
def test_batch_insert(self):
self.assertEqual(True, utilsMySql.deleteElements(tableName))
self.assertEqual(True, utilsRedis.deleteElements(tableName))
getConfig(config_file_path, DEBUG, reload=True)\
.set('General', 'numberofthreads', 0)\
.set('General', 'batchsize', 2)
rta_dltest_db_mysql = RTA_DLTEST_DB('mysql', config_file_path)
rta_dltest_db_redis = RTA_DLTEST_DB('redis-basic', config_file_path)
rta_dltest_db_mysql.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
rta_dltest_db_redis.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
self.assertEqual(0, utilsMySql.countElements(tableName))
self.assertEqual(0, utilsRedis.countElements(tableName))
rta_dltest_db_mysql.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
rta_dltest_db_redis.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
self.assertEqual(2, utilsMySql.countElements(tableName))
self.assertEqual(2, utilsRedis.countElements(tableName))
self.assertEqual(True, rta_dltest_db_mysql.waitAndClose())
self.assertEqual(True, rta_dltest_db_redis.waitAndClose())
def test_connection_with_wrong_password(self):
getConfig(config_file_path, DEBUG, reload=True)
getConfig(config_file_path, DEBUG).set('Redis', 'password', 'asdasd')
redisConn = RedisDBConnectorBASIC(config_file_path)
self.assertEqual(False, redisConn.connect())
self.assertEqual(False, redisConn.testConnection())
self.assertEqual(False, redisConn.close())
def test_execute_query_without_connection(self):
getConfig(config_file_path, DEBUG, reload=True)
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(
False,
mysqlConn.executeQuery('INSERT INTO ' + tableName +
' VALUES(1,2,3,4)'))
def test_multi_threading_mode_active(self):
getConfig(config_file_path, DEBUG, reload=True)\
.set('General','numberofthreads', 1)
rta_dltest_db = RTA_DLTEST_DB('mysql', config_file_path)
self.assertEqual(rta_dltest_db.multithreading, True)
rta_dltest_db.waitAndClose()
def test_insert_data_not_enough_data(self):
getConfig(config_file_path, DEBUG,
reload=True).set('General', 'batchsize', 1)
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(True, mysqlConn.connect())
self.assertEqual(
False,
mysqlConn.executeQuery('INSERT INTO ' + tableName +
' VALUES(1,2)'))
self.assertEqual(True, mysqlConn.close())
def synchronous_performance_test(batchsize, numberofthreads, numberOfIterationPerTest, utilsObj):
getConfig(configurationFilePath, False, reload=True)\
.set('General', 'batchsize', batchsize)\
.set('General', 'numberofthreads', numberofthreads)\
.set('Dtr', 'active', 'yes')\
.set('MySqlPipelineDatabase', 'active', 'no')
eventSecList = []
executionTimeList = []
for jj in range(numberOfIterationPerTest):
resetUniqueValue()
# Delete data
utilsObj.deleteElements(tableName)
elementsInserted = utils.countElements(tableName)
if elementsInserted != 0:
print("--> COUNT DATA CHECK FAILED!! Events in table: {}/0".format(elementsInserted))
exit()
sleep(1)
RTA_DLTEST = RTA_DLTEST_DB(database)
start_perf = perf_counter()
for i in range(numberOfEvents):
evt3data[i][0] = getUniqueValue()
RTA_DLTEST.insertEvent( *evt3data[i] )
RTA_DLTEST.waitAndClose()
end_perf = perf_counter()
executionTime = end_perf - start_perf
eventSec = numberOfEvents/executionTime
eventSecList.append(eventSec)
executionTimeList.append(executionTime)
print(".", end='', flush=True)
Perf = namedtuple('res', ['avg', 'stddev'])
avgES = mean(eventSecList)
stddevES = stdev(eventSecList)
ES = Perf(avgES, stddevES)
avgET = mean(executionTimeList)
stddevET = stdev(executionTimeList)
ET = Perf(avgET, stddevET)
print("\n{} +- {}\n{} +- {}".format(round(ES.avg,2), round(ES.stddev,2), round(ET.avg,2), round(ET.stddev,2)))
return ES
def test_modelname_does_not_exist(self):
getConfig(config_file_path, DEBUG, reload=True)
redisConn = RedisDBConnectorBASIC(config_file_path)
self.assertEqual(True, redisConn.connect())
self.assertEqual(
False, redisConn.insertData('idontexist', {
'a': 4,
'b': 2,
'c': 3
}))
self.assertEqual(True, redisConn.close())
def test_negative_batchsize(self):
getConfig(config_file_path, DEBUG, reload=True)
getConfig(config_file_path, DEBUG).set('General', 'batchsize', -1)
redisConn = RedisDBConnectorBASIC(config_file_path)
self.assertEqual(True, redisConn.connect())
self.assertEqual(
False, redisConn.insertData(tableName, {
'a': 4,
'b': 2,
'c': 3
}))
self.assertEqual(True, redisConn.close())
def test_closed_connection(self):
getConfig(config_file_path, DEBUG,
reload=True).set('General', 'numberofthreads', 0)
rta_dltest_db_mysql = RTA_DLTEST_DB('mysql', config_file_path)
rta_dltest_db_redis = RTA_DLTEST_DB('redis-basic', config_file_path)
self.assertEqual(True,
rta_dltest_db_mysql.dbConnector.testConnection())
self.assertEqual(True,
rta_dltest_db_redis.dbConnector.testConnection())
self.assertEqual(True, rta_dltest_db_mysql.waitAndClose())
self.assertEqual(True, rta_dltest_db_redis.waitAndClose())
self.assertEqual(False,
rta_dltest_db_mysql.dbConnector.testConnection())
self.assertEqual(False,
rta_dltest_db_redis.dbConnector.testConnection())
def test_insert_data_wrong_table(self):
getConfig(config_file_path, DEBUG,
reload=True).set('General', 'batchsize', 1)
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(True, mysqlConn.connect())
self.assertEqual(
False,
mysqlConn.insertData('lest_fable', {
'a': 1,
'b': 2,
'c': 3,
'd': 4
}))
self.assertEqual(True, mysqlConn.close())
def test_pythonic_usage(self):
self.assertEqual(True, utilsMySql.deleteElements(tableName))
self.assertEqual(True, utilsRedis.deleteElements(tableName))
getConfig(config_file_path, DEBUG, reload=True)
with RTA_DLTEST_DB('mysql', config_file_path) as rta_dltest_db_mysql:
rta_dltest_db_mysql.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
self.assertEqual(1, utilsMySql.countElements(tableName))
with RTA_DLTEST_DB('redis-basic',
config_file_path) as rta_dltest_db_redis:
rta_dltest_db_redis.insertEvent(*RTA_DLTEST_DB.getRandomEvent())
self.assertEqual(1, utilsRedis.countElements(tableName))
def test_insert_duplicate_data(self):
getConfig(config_file_path, DEBUG, reload=True)
redisConn = RedisDBConnectorBASIC(config_file_path)
self.assertEqual(True, redisConn.connect())
self.assertEqual(
True, redisConn.insertData(tableName, {
'a': 5,
'b': 2,
'c': 3
}))
self.assertEqual(
True, redisConn.insertData(tableName, {
'a': 5,
'b': 2,
'c': 3
}))
self.assertEqual(True, redisConn.close())
def test_batch_insert(self):
self.assertEqual(True, utils.deleteElements(tableName))
getConfig(config_file_path, DEBUG,
reload=True).set('General', 'batchsize', 2)
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(True, mysqlConn.connect())
self.assertEqual(
MySqlDBConnector.SUCCESS,
mysqlConn.insertData(tableName, {
'a': 1,
'b': 2,
'c': 3,
'd': 4
}))
self.assertEqual(1, mysqlConn.conn.in_transaction)
self.assertEqual(1, mysqlConn.commandsSent)
self.assertEqual(0, utils.countElements(tableName))
self.assertEqual(
MySqlDBConnector.SUCCESS_AND_COMMIT,
mysqlConn.insertData(tableName, {
'a': 5,
'b': 6,
'c': 7,
'd': 8
}))
self.assertEqual(0, mysqlConn.conn.in_transaction)
self.assertEqual(0, mysqlConn.commandsSent)
self.assertEqual(2, utils.countElements(tableName))
self.assertEqual(True, mysqlConn.close())
def test_close_not_opened_connection(self):
getConfig(config_file_path, DEBUG, reload=True)
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(False, mysqlConn.testConnection())
self.assertEqual(False, mysqlConn.close())
def test_connection_with_wrong_port(self):
getConfig(config_file_path, DEBUG,
reload=True).set('MySql', 'port', '9991')
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(False, mysqlConn.connect())
def test_connection_with_wrong_database(self):
getConfig(config_file_path, DEBUG,
reload=True).set('MySql', 'dbname', 'evttttest')
mysqlConn = MySqlDBConnector(config_file_path)
self.assertEqual(False, mysqlConn.connect())
def test_close_not_opened_connection(self):
getConfig(config_file_path, DEBUG, reload=True)
redisConn = RedisDBConnectorBASIC(config_file_path)
self.assertEqual(False, redisConn.close())
