def __init__(self, kafkaAddress, schemaRegistryAddress, testHost, testUser,
testDatabase, testSchema, testWarehouse, pk, pk_passphrase):
self.SEND_INTERVAL = 0.01 # send a record every 10 ms
self.VERIFY_INTERVAL = 60 # verify every 60 secs
self.MAX_RETRY = 10 # max wait time 10 mins
self.producer = Producer({'bootstrap.servers': kafkaAddress})
self.avroProducer = AvroProducer({
'bootstrap.servers':
kafkaAddress,
'schema.registry.url':
schemaRegistryAddress
})
reg = "[^\/]*snowflakecomputing" # find the account name
account = re.findall(reg, testHost)
if len(account) != 1 or len(account[0]) < 20:
print(
"Format error in 'host' field at profile.json, expecting account.snowflakecomputing.com:443"
)
pkb = parsePrivateKey(pk, pk_passphrase)
self.snowflake_conn = snowflake.connector.connect(
user=testUser,
private_key=pkb,
account=account[0][:-19],
warehouse=testWarehouse,
database=testDatabase,
schema=testSchema)
def __init__(self, kafkaAddress, schemaRegistryAddress,
kafkaConnectAddress, credentialPath, testVersion):
self.testVersion = testVersion
self.credentialPath = credentialPath
with open(self.credentialPath) as f:
credentialJson = json.load(f)
testHost = credentialJson["host"]
testUser = credentialJson["user"]
testDatabase = credentialJson["database"]
testSchema = credentialJson["schema"]
testWarehouse = credentialJson["warehouse"]
pk = credentialJson["encrypted_private_key"]
pk_passphrase = credentialJson["private_key_passphrase"]
self.TEST_DATA_FOLDER = "./test_data/"
self.httpHeader = {
'Content-type': 'application/json',
'Accept': 'application/json'
}
self.SEND_INTERVAL = 0.01 # send a record every 10 ms
self.VERIFY_INTERVAL = 60 # verify every 60 secs
self.MAX_RETRY = 120 # max wait time 120 mins
self.MAX_FLUSH_BUFFER_SIZE = 5000 # flush buffer when 10000 data was in the queue
self.kafkaConnectAddress = kafkaConnectAddress
self.schemaRegistryAddress = schemaRegistryAddress
self.kafkaAddress = kafkaAddress
self.adminClient = AdminClient({"bootstrap.servers": kafkaAddress})
self.producer = Producer({'bootstrap.servers': kafkaAddress})
self.avroProducer = AvroProducer({
'bootstrap.servers':
kafkaAddress,
'schema.registry.url':
schemaRegistryAddress
})
reg = "[^\/]*snowflakecomputing" # find the account name
account = re.findall(reg, testHost)
if len(account) != 1 or len(account[0]) < 20:
print(
datetime.now().strftime("%H:%M:%S "),
"Format error in 'host' field at profile.json, expecting account.snowflakecomputing.com:443"
)
pkb = parsePrivateKey(pk, pk_passphrase)
self.snowflake_conn = snowflake.connector.connect(
user=testUser,
private_key=pkb,
account=account[0][:-19],
warehouse=testWarehouse,
database=testDatabase,
schema=testSchema)