def verifyRetry(self):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError()
def verify(self, round):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError(
"Number of record in table is different from number of record sent"
)
# validate content of line 1
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.topic)).fetchone()
# "schema_id" is lost since they are using native avro converter
goldMeta = r'{"CreateTime":\d*,"key":{"bytes_val":"3q0=","dateTime":1234,"device":' \
r'{"deviceID":"555-4321","enabled":true},"double_array_val":' \
r'[0.3333333333333333,32.21,4.343243210000000e+08],"float_val":4321.432,' \
r'"int32_val":2147483647,"reading":321.321,"sint32_val":2147483647,"sint64_val":9223372036854775807,' \
r'"uint32_val":4294967295,"uint64_val":-1},"offset":\d*,"partition":\d*,"topic":"travis_correct_confluent_protobuf_protobuf....."}'
goldContent = r'{"bytes_val":"3q0=","dateTime":1234,"device":{"deviceID":"555-4321","enabled":true},"double_array_val":' \
r'[0.3333333333333333,32.21,4.343243210000000e+08],"float_val":4321.432,"int32_val":2147483647,' \
r'"reading":321.321,"sint32_val":2147483647,"sint64_val":9223372036854775807,"uint32_val":4294967295,"uint64_val":-1}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
self.driver.verifyStageIsCleaned(self.topic)
def verify(self, round):
# restart connector with different config
self.configIncreamental = self.configIncreamental + 1
if self.configIncreamental % 7 == 2:
configMap = {
"buffer.size.bytes": str(6000000 + self.configIncreamental)
}
self.driver.updateConnectorConfig(self.fileName,
self.connectorName, configMap)
elif self.configIncreamental % 7 == 3:
self.driver.restartConnector(self.connectorName)
elif self.configIncreamental % 7 == 4:
self.driver.pauseConnector(self.connectorName)
elif self.configIncreamental % 7 == 5:
self.driver.resumeConnector(self.connectorName)
elif self.configIncreamental % 7 == 6:
self.driver.deleteConnector(self.connectorName)
elif self.configIncreamental % 7 == 0:
self.driver.createConnector(self.getConfigFileName(),
self.nameSalt)
for t in range(self.curTest, self.topicNum):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topics[t])).fetchone()[0]
if res != self.partitionNum * self.recordNum * (round + 1):
raise RetryableError()
if self.curTest <= t:
self.curTest = t + 1
raise ResetAndRetry()
# after success, reset curTest for next round
self.curTest = 0
def verifyStageIsCleaned(self, connectorName, topicName=""):
if topicName == "":
topicName = connectorName
stageName = "SNOWFLAKE_KAFKA_CONNECTOR_{}_STAGE_{}".format(connectorName, topicName)
res = self.snowflake_conn.cursor().execute("list @{}".format(stageName)).fetchone()
if res is not None:
raise RetryableError("stage not cleaned up ")
def verify(self):
for t in range(self.curTest, self.topicNum):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topics[t])).fetchone()[0]
if res != self.partitionNum * self.recordNum * self.round:
raise RetryableError()
if self.curTest <= t:
self.curTest = t + 1
raise ResetAndRetry()
def verify(self):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError("Number of record in table is different from number of record sent")
# validate content of line 1
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.topic)).fetchone()
goldMeta = r'{"CreateTime":\d*,"offset":0,"partition":0,"topic":"travis_correct_string_json....."}'
goldContent = r'{"number":"0"}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
def verify(self):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.table)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError(
"Number of record in table is different from number of record sent"
)
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.table)).fetchone()
goldMeta = r'{"CreateTime":\d*,"key":{"int":"\d"},"offset":\d,"partition":\d,' \
r'"topic":"travis_correct_native_complex_smt....."}'
goldContent = r'{"c1":{"int":"\d"}}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
def verify(self):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError("Number of record in table is different from number of record sent")
# validate content of line 1
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.topic)).fetchone()
goldMeta = r'{"CreateTime":\d*,"key":[{"timestamp":\d*,"tweet":"Rock:Nerfpaper,scissorsisfine.",' \
r'"username":"******"},{"timestamp":\d*,"tweet":"Worksasintended.TerranisIMBA.",' \
r'"username":"******"}],"offset":0,"partition":0,"topic":"travis_correct_avro_avro....."}'
goldContent = r'{"timestamp":\d*,"tweet":"Rock:Nerfpaper,scissorsisfine.","username":"******"}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
def verify(self):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError(
"Number of record in table is different from number of record sent"
)
# validate content of line 1
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.topic)).fetchone()
goldMeta = r'{"CreateTime":\d*,"key":{"id":0},"key_schema_id":\d*,"offset":0,"partition":0,"schema_id":\d*,' \
r'"topic":"travis_correct_avrosr_avrosr....."}'
goldContent = r'{"firstName":"abc0","id":0,"time":1835}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
def verify(self):
# restart connector with different config
self.configIncreamental = self.configIncreamental + 1
if self.configIncreamental > 0:
configMap = {"buffer.size.bytes": str(6000000 + self.configIncreamental)}
self.driver.updateConnectorConfig(self.fileName, self.connectorName, configMap)
for t in range(self.curTest, self.topicNum):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topics[t])).fetchone()[0]
if res != self.partitionNum * self.recordNum:
raise RetryableError()
if self.curTest <= t:
self.curTest = t + 1
raise ResetAndRetry()
def verify(self, round):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError(
"Number of record in table is different from number of record sent"
)
# validate content of line 1
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.topic)).fetchone()
goldMeta = r'{"key":{"number":"0"},"offset":0,"partition":0}'
goldContent = r'{"number":"0"}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
self.driver.verifyStageIsCleaned(self.topic)
def verify(self, round):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError(
"Number of record in table is different from number of record sent"
)
# validate content of line 1, this way we ensure SMT is working
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.topic)).fetchone()
goldMeta = r'{"CreateTime":\d*,"offset":0,"partition":0,' \
r'"topic":"travis_correct_native_string_json_without_schema....."}'
goldContent = r'{"number":"0"}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
self.driver.verifyStageIsCleaned(self.topic)
def verify(self, round):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError("Number of record in table is different from number of record sent")
# validate content of line 1
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.topic)).fetchone()
# "schema_id" is lost since they are using native avro converter
goldMeta = r'{"CreateTime":\d*,"offset":0,"partition":0,' \
r'"topic":"travis_correct_native_string_avrosr....."}'
goldContent = r'{"firstName":"abc0","time":1835}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
self.driver.verifyStageIsCleaned(self.topic)
def verify(self, round):
res = self.driver.snowflake_conn.cursor().execute(
"SELECT count(*) FROM {}".format(self.topic)).fetchone()[0]
if res == 0:
raise RetryableError()
elif res != 100:
raise NonRetryableError("Number of record in table is different from number of record sent")
# validate content of line 1
if self.driver.testVersion != "5.5.0" and self.driver.testVersion != "2.5.0":
goldMeta = r'{"CreateTime":\d*,"headers":{"header1":"value1","header2":{}},"offset":0,"partition":0,"topic":"travis_correct_string_json....."}'
else:
goldMeta = r'{"CreateTime":\d*,"headers":{"header1":"value1","header2":[]},"offset":0,"partition":0,"topic":"travis_correct_string_json....."}'
res = self.driver.snowflake_conn.cursor().execute(
"Select * from {} limit 1".format(self.topic)).fetchone()
goldContent = r'{"number":"0"}'
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
self.driver.verifyStageIsCleaned(self.topic)