def run_kafka_server():
# TODO get the json file path
input_file = "police-department-calls-for-service.json" # same folder as this script
# TODO fill in blanks
# GK Note: I have completed the code template provided by Udacity.
# The provided code implies that the topic is autogenerated by KafkaProducer
# The provided code doesn't call the Kafka AdminClient to create the topic explicitly
# To experiment with number of Kafka partitions for performance, I create an admin client
bootstrap_servers = 'localhost:9092'
my_topic = "sf.police.calls"
admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
try:
print("attempting to create new topic")
topic_list = []
nb_partitions = 2
topic_list.append(
NewTopic(name=my_topic,
num_partitions=nb_partitions,
replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
except Exception:
print("topic already exists")
producer = producer_server.ProducerServer(
#producer = producer_server.ProducerServer(
input_file=input_file, # crime events file
topic=my_topic, # freely chosen
bootstrap_servers=bootstrap_servers, # standard choice for Kafka server
client_id="producer-01" # ID chosen for this producer
)
return producer
def run_kafka_producer():
"""
Create Kafka producer, check if relevant topic exists (if not create it) and start producing messages
"""
# load config
config = ConfigParser()
config.read(os.path.join(cur_path, "app.cfg"))
# start kafka producer
logger.info("Starting Kafka Producer")
producer = producer_server.ProducerServer(config)
# check if topic exists
logger.info("Creating topic...")
producer.create_topic()
# generate data
logger.info("Starting to generate data...")
try:
producer.generate_data()
except KeyboardInterrupt:
logging.info("Stopping Kafka Producer")
producer.close()
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="department.call.service.log",
bootstrap_servers="localhost:9092",
client_id="pd-call")
return producer
def run_kafka_server():
producer = producer_server.ProducerServer(
input_file=JSON_FILENAME,
topic_name=TOPIC_NAME,
bootstrap_servers=BROKER_URL,
)
return producer
def run_kafka_server():
input_file = f'{Path(__file__).parents[0]}/police-department-calls-for-service.json'
producer = producer_server.ProducerServer(
input_file=input_file,
topic=CRIME_DATA_TOPIC_NAME,
)
return producer
def run_kafka_server():
producer = producer_server.ProducerServer(
input_file="police-department-calls-for-service.json",
topic="udacity.police_service_calls.v1",
bootstrap_servers="localhost:9092",
client_id="sf-police-service-calls")
return producer
def run_kafka_server(config):
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic=config['topic'],
bootstrap_servers=config['brokers'],
client_id=config['client_id'])
return producer
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="sf_crime",
bootstrap_servers="localhost:9092",
client_id="org.sf.crime")
return producer
def run_kafka_server():
producer = producer_server.ProducerServer(
input_file=INPUT_FILE,
topic=TOPIC_NAME,
bootstrap_servers=BOOTSTRAP_SERVER,
client_id="sfpd.producer")
return producer
def run_kafka_server():
input_file = "./police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="org.sf.police.service-calls",
bootstrap_servers=["localhost:9092"],
client_id="0")
return producer
def run_kafka_server():
producer = producer_server.ProducerServer(
input_file=FILE,
topic="service.calls",
bootstrap_servers="localhost:9091",
client_id=None)
return producer
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(input_file=input_file,
topic=TOPIC_NAME,
bootstrap_servers=SERVER,
client_id=CLIENT)
return producer
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="com.sanfrancisco.police.calls_for_service.v1",
bootstrap_servers="localhost:9092",
client_id="sanfrancisco_import_police_calls_for_service")
return producer
def run_kafka_server():
input_file = 'police-department-calls-for-service.json'
producer = producer_server.ProducerServer(
input_file=input_file,
topic=TOPIC_NAME,
bootstrap_servers="localhost:9092",
client_id="consumer-1")
return producer
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic=KAFKA_TOPIC,
bootstrap_servers=[BOOTSTRAP_SERVER],
client_id="1"
)
return producer
def run_kafka_server():
input_file = DATA_FILE_PATH
producer = producer_server.ProducerServer(
input_file=input_file,
topic=KAFKA_TOPIC_NAME,
bootstrap_servers=KAFKA_BROKER_URL,
client_id="kafka-producer-from-json-file")
return producer
def run_kafka_server():
f_name = config.INPUT_FILE_NAME
input_file = utils.prepare_input_file(f_name)
topic_name = utils.get_topic_name(f_name)
producer = producer_server.ProducerServer(
input_file=input_file,
topic=topic_name,
bootstrap_servers=config.BOOTSTRAP_SERVERS)
return producer
def run_kafka_server():
input_file = 'police-department-calls-for-service.json'
producer = producer_server.ProducerServer(
input_file=input_file,
topic='com.udacity.police.calls',
bootstrap_servers='localhost:9092',
client_id='consumer-1')
return producer
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="sanfran.police.calls",
bootstrap_servers="localhost:9092",
)
return producer
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="com.udacity.sfcrimes.v1",
bootstrap_servers="localhost:9092",
client_id=None)
return producer
def run_kafka_server():
input_file = "./resources/police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="sfpd-calls",
bootstrap_servers="localhost:9092",
client_id="sfpd-project")
return producer
def run_kafka_server():
# get the json file path
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="org.udacity.assignment.crime-statistics",
bootstrap_servers="localhost:9092",
client_id="kafka-server-plain")
return producer
def run_kafka_server():
# TODO get the json file path
input_file = "./police-department-calls-for-service.json"
# TODO fill in blanks
producer = producer_server.ProducerServer(input_file=input_file,
topic=TOPIC_NAME,
bootstrap_servers=BROKER_URL,
client_id=CLIENT_ID)
return producer
def run_kafka_server():
input_file = "res/police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic=TOPIC,
bootstrap_servers=BOOTSTRAP_SERVERS,
client_id="sf-crimes"
)
return producer
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="crime-reports.v1",
bootstrap_servers="localhost:9092",
client_id="police-crime-kafka-producer"
)
return producer
def run_kafka_server():
input_file = "police-department-calls-for-service.json"
# Create a new produder
producer = producer_server.ProducerServer(
input_file=input_file,
topic="sf_crime_data.police.service.calls",
bootstrap_servers=BROKER_URL,
client_id="sf_data_crime.service.call.producer")
return producer
def run_kafka_server():
# TODO get the json file path
input_file = "police-department-calls-for-service.json"
# TODO fill in blanks
producer = producer_server.ProducerServer(input_file=input_file,
topic="com.udacity.project2",
bootstrap_servers="kafka0:19092",
client_id="me")
return producer
def run_kafka_server():
"""Writes police department service call events to Kafka topic from JSON file"""
input_file = "../data/police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic=KAFKA_TOPIC_NAME,
bootstrap_servers=BOOTSTRAP_SERVER,
client_id=CLIENT_ID)
return producer
def run_kafka_server():
# TODO get the json file path
input_file = "police-department-calls-for-service.json"
# TODO fill in blanks
producer = producer_server.ProducerServer(
input_file=input_file,
topic="org.sfpd.service.calls",
client_id="org.sfpd.service.calls.broker")
return producer
def run_kafka_server():
input_file = f"{Path(__file__).parents[0]}/police-department-calls-for-service.json"
producer = producer_server.ProducerServer(
input_file=input_file,
topic="sf.crime.statistics.topic",
bootstrap_servers="localhost:9092",
client_id=None #"socket.gethostname()"
)
return producer
