def clear_data_for_redispatching():
kafka_admin_client = KafkaAdminClient(
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS,
)
kafka_consumer = KafkaConsumer(
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS.split(";"),
)
kafka_admin_client.delete_topics(list(kafka_consumer.topics()))
print("--kafka cleared --")
if REDIS_PASSWORD == "":
redis_conn = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, password=None)
else:
redis_conn = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD)
redis_conn.flushdb()
print("--redis cleared --")
if not database_exists(str(SQLALCHEMY_DATABASE_URI)):
print("Failed, the database does not exist.")
return False
engine = create_engine(str(SQLALCHEMY_DATABASE_URI))
db_session = sessionmaker(bind=engine)
session = db_session()
session.execute("delete from job_scheduled_secondary_workers;")
session.execute("update job set planning_status = 'U';")
session.execute("update team set latest_env_kafka_offset=0;")
session.execute("commit;")
print("--postgres DB cleared --")
def test_produce_and_consume(self):
"""
It first creates 10 messages. The it consumes them. It is successful, if all messages are consumed.
:return: -
"""
topic = 'test_4'
topic_partition = TopicPartition(topic='test_4', partition=0)
msg = b'this is a message'
# publish 10 events to the topic 'wat'
producer = KafkaProducer(bootstrap_servers=bootstrap_servers)
for _ in range(10):
producer.send(topic, msg)
# consume all previous events, that where published to the topic 'wat'
consumer = KafkaConsumer(topic,
bootstrap_servers=bootstrap_servers,
auto_offset_reset='earliest')
events = consumer.poll(1000)[topic_partition]
n_events = len(events)
# the events is a list of events that must not be empty
assert (n_events > 0)
# the last event must (most likely) have the value 'this is a message'
assert (events[n_events - 1].value == msg)
client = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
client.delete_topics([topic])
def run(self):
try:
consumer = KafkaConsumer(self.response_topic,
bootstrap_servers=self.servers,
client_id=random_string())
self.logger.info("manifest consumer on {0} kafka topic".format(
self.response_topic))
for message in consumer:
try:
json_str = message.value
json_msg = json.loads(json_str)
if is_close_msg(json_msg):
print(json_str)
break
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback,
limit=20,
file=sys.stdout)
self.logger.error(str(exc_type))
self.logger.error(str(exc_value))
break
consumer.close()
admin = KafkaAdminClient(bootstrap_servers=self.servers)
admin.delete_topics([self.response_topic])
admin.close()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=20, file=sys.stdout)
self.logger.error(str(exc_type))
self.logger.error(str(exc_value))
def delete_topics(topics, time_out=10):
# noinspection PyBroadException
config = get_basic_utilities().get(CONFIG)
kafka_config = config['kafka']
server = f"{kafka_config['host']}:{kafka_config['port']}"
admin_client = KafkaAdminClient(bootstrap_servers=server)
consumer = KafkaConsumer(bootstrap_servers=server)
active_topics = [topic for topic in consumer.topics() if topic in topics]
admin_client.delete_topics(topics=active_topics)
stop = threading.Event()
all_deleted = False
def is_deleted():
nonlocal all_deleted
while not stop.is_set():
current_topics = consumer.topics()
for topic in active_topics:
if topic in current_topics:
stop.wait(timeout=1)
break
else:
stop.set()
all_deleted = True
thread = threading.Thread(name='delete-topics', target=is_deleted)
thread.start()
stop.wait(timeout=time_out)
stop.set()
thread.join()
return all_deleted
class TestElasticProducer(object):
def setup_class(self):
self.admin = KafkaAdminClient(bootstrap_servers='localhost:9092')
self.index = ElasticIndex('test-elastic-producer', 'doc')
self.index.index_into({'test': 1}, 0)
self.index.index_into({'test': 2}, 1)
self.index.index_into({'test': 3}, 2)
self.index.index_into({'test': 4}, 3)
self.index.index_into({'test': 5}, 4)
self.producer = ElasticProducer('configs/elastic/test_elastic_producer_producer.yml')
self.consumer = SimpleConsumer('configs/elastic/test_elastic_producer_consumer.yml')
def teardown_class(self):
self.consumer.close()
self.admin.delete_topics(['test-elastic-producer'])
self.admin.close()
self.index.delete()
#@pytest.mark.skip()
def test_produce(self):
self.producer.process()
key, message = self.consumer.consume()
assert key == '0'
assert message == '{"test": 1}'
def del_topic(args):
admin = KafkaAdminClient(bootstrap_servers=[args.broker])
try:
admin.delete_topics(topics=args.topics)
except UnknownTopicOrPartitionError:
pass
admin.close()
def run(self):
try:
self.logger.info("starting rosbag_consumer:{0}".format(
self.response_topic))
rospy.init_node("mozart_rosbag_{0}".format(random_string(6)))
consumer = KafkaConsumer(self.response_topic,
bootstrap_servers=self.servers,
client_id=random_string())
if self.s3:
self.s3_reader = S3Reader(self.s3_read_req, self.s3_read_resp)
self.s3_deleter = S3Deleter(self.s3_delete_req)
self.s3_reader.start()
self.s3_deleter.start()
for msg in consumer:
try:
json_str = msg.value
json_msg = json.loads(json_str)
if is_close_msg(json_msg):
print(json_str)
break
self.publish_bag(json_msg)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback,
limit=20,
file=sys.stdout)
print(str(e))
break
if self.s3:
self.read_s3(drain=True)
self.s3_read_req.put("__close__")
self.s3_reader.join(timeout=2)
if self.s3_reader.is_alive():
self.s3_reader.terminate()
self.s3_delete_req.put("__close__")
time.sleep(5)
self.s3_deleter.join(timeout=2)
if self.s3_deleter.is_alive():
self.s3_deleter.terminate()
else:
for dir in self.clean_up:
shutil.rmtree(dir, ignore_errors=True)
consumer.close()
admin = KafkaAdminClient(bootstrap_servers=self.servers)
admin.delete_topics([self.response_topic])
admin.close()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=20, file=sys.stdout)
print(str(e))
def clear_kafka():
kafka_admin_client = KafkaAdminClient(
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS, )
kafka_consumer = KafkaConsumer(
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS.split(";"), )
kafka_admin_client.delete_topics(list(kafka_consumer.topics()))
print("--kafka cleared --")
class Admin:
def __init__(self):
self.client = KafkaAdminClient(bootstrap_servers=app.config['KAFKA_URL'])
def create_topics(self, topics):
topics = [NewTopic(topic, 1, 1) for topic in topics]
self.client.create_topics(new_topics=topics)
def delete_topics(self, topics):
self.client.delete_topics(topics=topics)
def clean_topics():
kafka_admin_client = KafkaAdminClient(
bootstrap_servers=['192.168.56.101:29094'])
topics = kafka_admin_client.list_topics()
my_topics = []
for topic in topics:
if "owid-covid" in topic:
print(topic)
my_topics.append(topic)
kafka_admin_client.delete_topics(my_topics)
class AdminClient(KafkaPython):
def __init__(self, bootstrap_servers=None, **kwargs):
super().__init__(servers=bootstrap_servers)
admin_client_config.update(kwargs)
self.engine = KafkaAdminClient(
bootstrap_servers=self._bootstrap_servers,
client_id=self._client_id,
request_timeout_ms=self._request_timeout_ms,
**admin_client_config)
def create_topics(self, topic_list: list):
new_topic = []
for k, v in enumerate(topic_list):
new_topic.append(
NewTopics(name=v['name'],
num_partitions=v['num_partitions'],
replication_factor=v['replication_factor'],
replica_assignments=v['replica_assignments'],
topic_configs=v['topic_configs']))
if not Consumer().get_user_topics().intersection(
{item['name']
for i, item in enumerate(topic_list)}):
try:
self.engine.create_topics(new_topic, **create_topic_config)
except KafkaError as e:
_logger.error(e)
self.engine.close()
else:
_logger.error(
self._logMsg(create_topic_fail_code, self._client_id,
'topic重复'))
return
def delete_topics(self, topic: list):
if Consumer().get_user_topics().intersection(set(topic)):
try:
self.engine.delete_topics(topic, self._request_timeout_ms)
except KafkaError as e:
_logger.error(
self._logMsg(delete_topic_fail_code, self._client_id,
'删除的topic失败:%s' % e))
self.engine.close()
else:
_logger.error(
self._logMsg(delete_topic_fail_code, self._client_id,
'需删除的topic不存在'))
return
def clear_all_data():
kafka_admin_client = KafkaAdminClient(
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS,
)
kafka_consumer = KafkaConsumer(
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS.split(";"),
)
kafka_admin_client.delete_topics(list(kafka_consumer.topics()))
print("--kafka cleared --")
if REDIS_PASSWORD == "":
redis_conn = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, password=None)
else:
redis_conn = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD)
redis_conn.flushdb()
print("--redis cleared --")
if not database_exists(str(SQLALCHEMY_DATABASE_URI)):
print("Failed, the database does not exist.")
return False
engine = create_engine(str(SQLALCHEMY_DATABASE_URI))
db_session = sessionmaker(bind=engine)
session = db_session()
session.execute("delete from event;")
session.execute("delete from assoc_job_tags;")
session.execute("delete from job_scheduled_secondary_workers;")
session.execute("delete from job;")
# session.execute("delete from worker_absence;")
# session.execute("delete from appointment;")
session.execute("delete from worker;")
session.execute("delete from location;")
# session.execute("update team set latest_env_kafka_offset=0;")
session.execute("delete from team;")
session.execute("delete from tag;")
session.execute("delete from service_plugin;")
session.execute("delete from plugin;")
session.execute("delete from service;")
session.execute("delete from dispatch_user;")
session.execute("commit;")
print("--postgres DB cleared --")
def test_runner(self):
"""
Generalized producer consumer test.
Assuming there is an empty test kafka rollout locally at port 9092
"""
test_run_id = str(uuid.uuid1())
test_config = StatsProcessorConfig()
test_config.servers = ['localhost:9092']
test_config.topics = ['test-topic']
test_config.client_id = 'test-client'
# Remove existing topics
admin_client = KafkaAdminClient(bootstrap_servers=test_config.servers,
client_id='test-admin')
kafka_topics = admin_client.list_topics()
existing_topics = set(test_config.topics).intersection(kafka_topics)
if len(existing_topics) > 0:
admin_client.delete_topics(existing_topics)
# Initialize the database
local_db_mngr = LocalDataManager(self.test_dir)
local_db_mngr.initialize_db()
runner = StatsCollectorRunner(self.test_dir, test_config)
runner.initialize_runner()
test_producer = TestEventsProducer(kafka_servers=test_config.servers,
test_topic=test_config.topics[0],
test_run_id=test_run_id)
test_producer.start()
runner.start()
time.sleep(10) # run for 10 seconds
runner.stop()
print('Runner stopping')
runner.join()
print('Runner exited')
test_producer.stop()
print('Producer stopping')
test_producer.join()
print('Producer exited')
with local_db_mngr as bench:
print(bench.get_run_nodes(test_run_id))
def test_delete_topics(self):
"""
Caution! This test is supposed to run in a test environment. Do not use it in a production environment,
as it is supposed to delete all topics! The test environment can be started with
tests/resources docker-compose.yml
:return: -
"""
# get all topics...
consumer = KafkaConsumer(bootstrap_servers=bootstrap_servers)
topics = consumer.topics()
# ...and delete them
client = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
client.delete_topics(topics)
# ...now check if they were all deleted
topics = consumer.topics()
assert (len(topics) == 0)
class TestSimpleElasticConsumer(object):
def setup_class(self):
self.admin = KafkaAdminClient(bootstrap_servers='localhost:9092')
self.consumer = SimpleElasticConsumer('configs/elastic/elastic-consumer-simple-test.yml')
self.bulk_consumer = BulkElasticConsumer('configs/elastic/elastic-consumer-bulk-test.yml')
self.producer = LineProducer("configs/elastic/json-lines-producer.yml")
self.producer.process()
def teardown_class(self):
self.admin.delete_topics(['json'])
self.consumer.close()
#@pytest.mark.skip()
def test_consume(self):
result = self.consumer.consume()
assert result
# @pytest.mark.skip()
def test_consume_bulk(self):
result = self.bulk_consumer.consume()
assert result
def cleanup():
filter = flask.request.args.get('filter')
if filter == None:
filter = ""
consumer = openConsumer(brokerArray)
admin_client = KafkaAdminClient(bootstrap_servers=brokerArray)
topics = consumer.topics()
returnTopicCount = 0
for n in topics:
log("Found Topic - " + n)
if filter in n:
tp = TopicPartition(n, 0)
consumer.assign([tp])
lastOffset = consumer.end_offsets([tp])[tp]
log("Found Topic - " + str(lastOffset))
readOffset = 0
if lastOffset > 0:
readOffset = lastOffset - 1
consumer.seek(tp, readOffset)
messages = consumer.poll(timeout_ms=1000)
log("Found Topic - " + str(messages))
if tp in messages:
log("Removing Topic - " + str(messages[tp][0].timestamp))
if messages[tp][0].timestamp < (
(time.time() - DEFAULT_CLEANUP_TIME) * 1000):
log("Deleting Topic - " + n)
admin_client.delete_topics([n])
returnTopicCount += 1
elif (readOffset == 0) or (len(messages) == 0):
log("Deleting Topic - " + n)
admin_client.delete_topics([n])
returnTopicCount += 1
return "Deleted " + str(returnTopicCount) + " Topics"
class AdminClient():
def __init__(self, kafkaBroker):
self.admin = KafkaAdminClient(bootstrap_servers=kafkaBroker)
def CreateTopics(self, topics, numpartitions=3):
""" Create topics """
new_topics = [
NewTopic(topic, num_partitions=numpartitions, replication_factor=1)
for topic in topics
]
# Call create_topics to asynchronously create topics, a dict
# of <topic,future> is returned.
fs = self.admin.create_topics(new_topics)
for topic in topics:
print("Topic {} created".format(topic))
'''
# Wait for operation to finish.
# Timeouts are preferably controlled by passing request_timeout=15.0
# to the create_topics() call.
# All futures will finish at the same time.
for topic, f in fs.items():
try:
f.result() # The result itself is None
print("Topic {} created".format(topic))
except Exception as e:
print("Failed to create topic {}: {}".format(topic, e))
'''
def DeleteTopics(self, topics):
""" delete topics """
# Call delete_topics to asynchronously delete topics, a future is returned.
# By default this operation on the broker returns immediately while
# topics are deleted in the background. But here we give it some time (30s)
# to propagate in the cluster before returning.
#
# Returns a dict of <topic,future>.
fs = self.admin.delete_topics(topics, timeout_ms=30)
# Wait for operation to finish.
for topic in topics:
print("Topic {} deleted".format(topic))
'''
def test_admin_client(self):
"""
This test verifies that Kafka Admin Client can still be used to manage Kafka.
"""
admin_client = KafkaAdminClient(
bootstrap_servers=KafkaBrokerIntegrationTest.kafka_address())
# Create a topic with 3 partitions.
new_topic_spec = NewTopic(name='test_admin_client',
num_partitions=3,
replication_factor=1)
create_response = admin_client.create_topics([new_topic_spec])
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
# Alter topic (change some Kafka-level property).
config_resource = ConfigResource(ConfigResourceType.TOPIC,
new_topic_spec.name,
{'flush.messages': 42})
alter_response = admin_client.alter_configs([config_resource])
error_data = alter_response.resources
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0][0], 0)
# Add 2 more partitions to topic.
new_partitions_spec = {new_topic_spec.name: NewPartitions(5)}
new_partitions_response = admin_client.create_partitions(
new_partitions_spec)
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
# Delete a topic.
delete_response = admin_client.delete_topics([new_topic_spec.name])
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
self.metrics.collect_final_metrics()
self.metrics.assert_metric_increase('create_topics', 1)
self.metrics.assert_metric_increase('alter_configs', 1)
self.metrics.assert_metric_increase('create_partitions', 1)
self.metrics.assert_metric_increase('delete_topics', 1)
class Admin:
def __init__(self, bootstrap_servers):
self.admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
def create_topic(self, topic):
try:
topic = NewTopic(name=topic,
num_partitions=1,
replication_factor=1)
self.admin.create_topics([topic], timeout_ms=2000)
except Exception:
pass
def delete_topic(self, topic):
res = self.admin.delete_topics([topic], 2000)
def close(self):
self.admin.close()
def main(args):
if args.dry:
print("--- This a DRY run. No topic will be deleted! ---")
# set up the bootstrap servers for kafka client
bootstrap_servers = [x for x in args.broker_list.split(",")]
kafka_consumer = KafkaConsumer(bootstrap_servers=bootstrap_servers)
kafka_topics = kafka_consumer.topics()
kafka_admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
mongo_uri = 'mongodb://{0}/?replicaSet=rs0'.format(",".join([x for x in args.mongo_list.split(",")]))
topics_col = MongoClient(mongo_uri).get_database(name="argo_msg").get_collection(name="topics")
ams_topics = set()
for top in topics_col.find():
ams_topics.add("{0}.{1}".format(top["project_uuid"], top["name"]))
deleted_topics_count = 0
topics_to_be_deleted = kafka_consumer.topics().difference(ams_topics)
for top_to_del in topics_to_be_deleted:
print("Marking topic: " + str(top_to_del) + " for deletion (X)")
if not args.dry:
try:
print(kafka_admin_client.delete_topics(topics=[top_to_del]))
deleted_topics_count += 1
print("---------------------------------------------------")
except UnknownTopicOrPartitionError as e:
print("Could not delete topic {0}. Exception: {1}"
.format(top_to_del, str(e.message)))
print("---------------------------------------------------")
continue
print("Total Kafka topics: {0}".format(len(kafka_topics)))
print("Total AMS topics: {0}".format(len(ams_topics)))
print("Total Marked topics: {0}".format(len(topics_to_be_deleted)))
print("Total Deleted topics: {0}".format(deleted_topics_count))
class AdminClient(object):
def __init__(self, **configs):
self._instance = KafkaAdminClient(**configs)
def create_topic(self, name: str, num_partitions: int,
replication_factor: int, **configs: Dict[str, str]):
"""
See https://kafka.apache.org/documentation/#topicconfigs for topic configs options.
:param name: The name of the new topic.
:param num_partitions: The number of partitions within this topic.
:param replication_factor: The number of replicas to be created.
:param configs: Configs as a dict {str: str}.
:return: Version of CreateTopicResponse class.
"""
topic = NewTopic(name,
num_partitions,
replication_factor,
topic_configs=configs)
return self._instance.create_topics([topic])
def delete_topic(self, name: str):
return self._instance.delete_topics([name])
from kafka import KafkaAdminClient
import os
admin = KafkaAdminClient(bootstrap_servers='localhost:9092')
os.system('pkill -9 -f app.py')
os.system('pkill -9 -f consumer.py')
os.system('pkill -9 -f producer.py')
admin.delete_topics(['dashboard1', 'clickhouse1'])
admin.close()
#os.system('sudo docker stop $(sudo docker ps -a -q)')
type=int,
default="10")
args = parser.parse_args()
kafka_brokers = args.kafka_broker
number_of_msgs = args.number_of_messages
topic = args.topic
consumer_timeout_ms = args.consumer_timeout_ms
msg_send = []
msg_recv = []
admin = KafkaAdminClient(bootstrap_servers=kafka_brokers)
try:
admin.delete_topics([topic], timeout_ms=5000)
except:
print("Cannot delete topic, might not exist")
consumer = KafkaConsumer(bootstrap_servers=kafka_brokers,
auto_offset_reset='earliest',
consumer_timeout_ms=consumer_timeout_ms)
producer = KafkaProducer(bootstrap_servers=kafka_brokers)
consumer.subscribe(topic)
for i in range(number_of_msgs):
now_start = round(monotonic_ns())
producer.send(
topic,
bytes(
#!/usr/bin/env python # -*- coding: utf-8 -*- from kafka import KafkaAdminClient servers = ['192.168.5.110:9092'] adminClient = KafkaAdminClient(bootstrap_servers=servers) adminClient.delete_topics(['test']) print(adminClient.list_consumer_groups()) adminClient.close()
def _delete_kafka_topic(topic_name):
"""Delte the given kafka topic running locally."""
kafka_client = KafkaAdminClient(bootstrap_servers='localhost:9092')
kafka_client.delete_topics([topic_name])
def delete_topics(topic='mykafka'):
admin = KafkaAdminClient(bootstrap_servers='6.86.2.170:9092')
topics = list_topics()
if topic in topics:
admin.delete_topics(topics=[topic])
def main(argv):
with open('register.json') as f:
config = json.load(f)
with open('tpc-config.json') as f:
tpcconfig = json.load(f)
if (len(argv) > 0):
bootstrapserver = argv[0]
print(config['config']['connector.class'])
print(config['name'])
config['name'] = 'tpc-connector'
print(config['name'])
config['config']['database.history.kafka.topic'] = 'tpc-test'
databasetype = config['config']['connector.class']
connectiontype = config['config']['connector.class'].split('.')[3]
print(databasetype)
print(connectiontype)
table = tpcconfig['jdbc'][connectiontype].get('table')
if table == None:
table = 'TPC.TEST'
config['config']['table.include.list'] = table
lowercase = tpcconfig['jdbc'][connectiontype].get('lowercase')
if lowercase == None:
lowercase = False
conn = getjdbcconnection(config, tpcconfig, connectiontype)
initsql(conn, config, tpcconfig)
createTPCTable(conn, config, tpcconfig)
enablecdctablesql(conn, config, tpcconfig)
print('============')
print('get status tpc connector')
resp = requests.get(
'http://' + tpcconfig['debezium.connect.server'] + '/connectors/tpc-connector/status', verify=False)
print(resp.content)
print(resp.status_code)
retvalue = json.loads(resp.content)
print(retvalue)
print('============')
resp = requests.delete(
'http://' + tpcconfig['debezium.connect.server'] + '/connectors/tpc-connector', verify=False)
print(resp.content)
print(resp.status_code)
if (resp.status_code == 404):
print('tpc-connector not exists')
else:
print('tpc-connector deleted')
pass
databaseservername = config['config']['database.server.name']
topicname = databaseservername + '.' + table
historybootstrapserver = config['config'].get('database.history.kafka.bootstrap.servers')
if historybootstrapserver != None:
bootstrapserver = historybootstrapserver.split(",")
# check integrated test ( all in one docker)
if bootstrapserver == 'kafka:9092':
print(bootstrapserver)
kafkaadmin = KafkaAdminClient(bootstrap_servers=bootstrapserver)
try:
kafkaadmin.delete_topics(
[topicname], 30)
except:
print(topicname + ' TOPIC not exists')
else:
print(topicname + ' TOPIC deleted')
if historybootstrapserver != None:
try:
kafkaadmin.delete_topics(
[config['config']['database.history.kafka.topic']], 30)
except:
print(config['config']['database.history.kafka.topic'] +
' TOPIC not exists')
else:
print(config['config']
['database.history.kafka.topic'] + ' TOPIC deleted')
# start tpc connector
print('start tpc connector')
resp = requests.post(
'http://' + tpcconfig['debezium.connect.server'] + '/connectors', headers={'content-type': 'application/json'}, data=json.dumps(config), verify=False)
print(resp.content)
print(resp.status_code)
retvalue = json.loads(resp.content)
print(retvalue)
while resp.status_code > 200:
print('get status tpc connector')
resp = requests.get(
'http://' + tpcconfig['debezium.connect.server'] + '/connectors/tpc-connector/status', verify=False)
print(resp.content)
print(resp.status_code)
retvalue = json.loads(resp.content)
print(retvalue)
time.sleep(1)
conn.jconn.setAutoCommit(False)
for x in range(len(tpcconfig['tpc']['commit.intervals'])):
print(tpcconfig['tpc']['commit.intervals'][x])
curs = conn.cursor()
for y in range(int(tpcconfig['tpc']['count'])):
curs.execute(tpcconfig['sql']['insert'])
if ((y % (tpcconfig['tpc']['commit.intervals'][x])) == (tpcconfig['tpc']['commit.intervals'][x] - 1)):
conn.commit()
conn.commit()
topicexport(bootstrapserver, topicname, int(
tpcconfig['tpc']['count']), tpcconfig['tpc']['commit.intervals'][x])
kafkaadmin.delete_topics([topicname], 30)
print('Wait 30 second for TOPIC clean up')
time.sleep(30)
used_topics = (
"topic_oferte",
"topic_rezultat",
"topic_oferte_procesate",
"topic_notificare_procesor_mesaje",
)
# se sterg topic-urile, daca exista deja
print("Se sterg topic-urile existente...")
kafka_topics = admin.list_topics()
for topic in kafka_topics:
if topic in used_topics:
print("\tSe sterge {}...".format(topic))
admin.delete_topics(topics=[topic], timeout_ms=2000)
# se asteapta putin ca stergerea sa aiba loc
time.sleep(2)
# se creeaza topic-urile necesare aplicatiei
print("Se creeaza topic-urile necesare:")
lista_topicuri = [
NewTopic(name=used_topics[0], num_partitions=4, replication_factor=1),
NewTopic(name=used_topics[1], num_partitions=1, replication_factor=1),
NewTopic(name=used_topics[2], num_partitions=1, replication_factor=1),
NewTopic(name=used_topics[3], num_partitions=1, replication_factor=1)
]
for topic in lista_topicuri:
print("\t{}".format(topic.name))
admin.create_topics(lista_topicuri, timeout_ms=3000)
def delete_topics(topic='test'):
admin = KafkaAdminClient(bootstrap_servers='localhost:9092')
topics = list_topics()
if topic in topics:
admin.delete_topics(topics=[topic])
class TestLineProducer(object):
def setup_class(self):
self.admin = KafkaAdminClient(bootstrap_servers='localhost:9092')
self.producer = LineProducer('configs/lines/producer.yml')
self.producer_gz = LineProducer('configs/lines/producer_gz.yml')
self.producer_bz2 = LineProducer('configs/lines/producer_bz2.yml')
self.consumer = SimpleConsumer('configs/lines/consumer.yml')
self.consumer_gz = SimpleConsumer('configs/lines/consumer_gz.yml')
self.consumer_bz2 = SimpleConsumer('configs/lines/consumer_bz2.yml')
self.ntriples_producer = SortedNTriplesCollectorProducer(
'configs/nt/producer.yml')
self.ntriples_consumer = BulkElasticConsumer('configs/nt/consumer.yml')
def teardown_class(self):
self.consumer.close()
self.consumer_gz.close()
self.consumer_bz2.close()
try:
self.admin.delete_topics(['test-lines-gz'])
except UnknownTopicOrPartitionError:
pass
try:
self.admin.delete_topics(['test-lines'])
except UnknownTopicOrPartitionError:
pass
try:
self.admin.delete_topics(['test-lines-bz2'])
except UnknownTopicOrPartitionError:
pass
try:
self.admin.delete_topics(['test-sorted-nt-resource'])
except UnknownTopicOrPartitionError:
pass
self.admin.close()
def test_ntriples_producer(self):
self.ntriples_producer.process()
assert self.ntriples_consumer.consume()
#@pytest.mark.skip()
def test_produce(self):
self.producer.process()
key = None
message = None
while key is None and message is None:
key, message = self.consumer.consume()
assert key == '0'
assert message == "This is a line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer.consume()
assert key == '1'
assert message == "and another line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer.consume()
assert key == '2'
assert message == "a third line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer.consume()
assert key == '3'
assert message == "a forth line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer.consume()
assert key == '4'
assert message == "a lot of lines now"
#@pytest.mark.skip("Currently way too slow")
def test_produce_gz(self):
self.producer_gz.process()
key = None
message = None
while key is None and message is None:
key, message = self.consumer_gz.consume()
assert key == '0'
assert message == "This is a line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer_gz.consume()
assert key == '1'
assert message == "and another line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer_gz.consume()
assert key == '2'
assert message == "a third line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer_gz.consume()
assert key == '3'
assert message == "a forth line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer_gz.consume()
assert key == '4'
assert message == "a lot of lines now"
# @pytest.mark.skip("Currently way too slow")
def test_produce_bz2(self):
self.producer_bz2.process()
key = None
message = None
while key is None and message is None:
key, message = self.consumer_bz2.consume()
assert key == '0'
assert message == "This is a line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer_bz2.consume()
assert key == '1'
assert message == "and another line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer_bz2.consume()
assert key == '2'
assert message == "a third line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer_bz2.consume()
assert key == '3'
assert message == "a forth line"
key = None
message = None
while key is None and message is None:
key, message = self.consumer_bz2.consume()
assert key == '4'
assert message == "a lot of lines now"
