def __init__(self, host, port, schema_path, topic, nbmsg, consumer_timeout):
self.topic = topic
self.nbmsg = nbmsg
self.sent_msg = 0
self.host = host
self.port = port
self.sent = [-100] * self.nbmsg
self.rcv = [-100] * self.nbmsg
self.runtag = str(random.randint(10, 100000))
try:
self.broker = KafkaClient("%s:%d" % (self.host, self.port))
except:
raise ValueError(
"KafkaClient (%s:%d) - init failed" % (self.host, self.port))
try:
self.producer = SimpleProducer(self.broker)
except:
raise ValueError(
"SimpleProducer (%s:%d) - init failed" % (self.host, self.port))
try:
self.consumer = SimpleConsumer(
self.broker, "testbot", topic, iter_timeout=consumer_timeout)
except:
raise ValueError(
"SimpleConsumer (%s:%d) - init failed" % (self.host, self.port))
try:
self.schema = avro.schema.parse(open(schema_path).read())
except:
raise ValueError(
"Prod2Cons load schema (%s) - init failed" % (schema_path))
def run(self):
client = KafkaClient("localhost:9092")
producer = SimpleProducer(client)
ctime_start = int(time.mktime(time.strptime("30-12-2010 14:00", "%d-%m-%Y %H:%M")))
for i in range(1):
ctime_length = 60
ctime_interval = 60*60
print range(ctime_start,
ctime_start+ctime_length,
ctime_interval )
ctime_starts = [ctime_start]
for i,photo_id in enumerate(GetPhotoIDs_batch_iter(ctime_starts,
interval=ctime_interval)):
print i, ctime_start, photo_id
producer.send_messages('flickr-photo_id-dist', photo_id)
#print photo_id
time.sleep(0.3)
ctime_start += ctime_interval
time.sleep(1)
def run(self):
client = KafkaClient(settings.KAFKA_SERVER)
producer = SimpleProducer(client)
while True:
producer.send_messages('heartbeats', """{"id":"yes-is-a-fake-uuide", "status": 200, "serviceName":"chewit_cam" }""")
time.sleep(5)
class KafkaMetrics(object):
def __init__(self, broker_list, kafka_topic):
logging.basicConfig(level=logging.INFO)
self.log = logging.getLogger('druid-kafka-metrics')
self.log.info("Kafka (brokers=%s, topic=%s)" %(broker_list, kafka_topic))
client = KafkaClient(broker_list)
self.producer = SimpleProducer(client)
self.msg_count = 0
self.kafka_topic = kafka_topic
@cherrypy.expose
@cherrypy.tools.json_out()
@cherrypy.tools.json_in()
def metrics(self):
messages = cherrypy.request.json
for message in messages:
self.msg_count += 1
self.log.debug("%s - %s" % (self.msg_count, str(message)))
self.producer.send_messages(self.kafka_topic, json.dumps(message))
if self.msg_count % 100 == 0 :
self.log.info("%s messages processed." % (self.msg_count, ))
return "{'code':200}"
def run(self):
#client = KafkaClient("localhost:9092")
client = KafkaClient("kafka_host:9092")
producer = SimpleProducer(client)
while True:
try:
messages = []
for i in xrange(1, 10):
messageStr = SelfGeneratedMessage().asJson()
logger.debug('Generated message: %s', messageStr)
messages.append(messageStr)
producer.send_messages('test', *messages)
# producer.send_messages('test', '{"publisher": "publisher-id", "time": "2015-11-03 15:03:30.352", "readings": [ 1, 1,1,1,1,1,1,1,1,1,1,1,4,3,3,3,32,2,1,1,1,1]}')
time.sleep(1)
except LeaderNotAvailableError as e:
logging.exception('LeaderNotAvailableError')
time.sleep(10)
except KafkaUnavailableError as e:
logging.exception('KafkaUnavailableError')
time.sleep(30)
except ConnectionError as e:
logging.exception('ConnectionError')
time.sleep(60)
except KafkaError as e:
logging.exception('KafkaError')
time.sleep(60)
except Exception as e:
logging.exception('Exception')
time.sleep(60)
def run():
db = getdb()
writer = WriterProcess()
writer.start()
client = KafkaClient(config.KAFKA_HOST)
producer = SimpleProducer(client)
timings = []
for i in range(1000, 2000):
key = random_key()
start = time.clock()
message = IWAMessage(key, i)
# print('Sending: %s' % message.dumps())
producer.send_messages(config.KAFKA_TOPIC, message.dumps())
# print('Queued: %s => %s' % (message.key, message.value))
while True:
with db.begin() as txn:
if txn.get(key):
timings.append(time.clock() - start)
break
writer.terminate()
print('Average write time: %s' % (sum(timings) / len(timings)))
print('Total write time: %s' % (sum(timings)))
class Producer():#threading.Thread):
daemon = True
# Initializes producer with commandline options
def __init__(self,options):
self.host = options.host
self.port = options.port
self.topic = options.topic
self.logfile = options.logfile
self.metadata = options.metadata
self.batch_size = options.batch_size
self.delay = options.delay
self.pending_messages = []
# Formats message to be sent to kafka
def create_message_data(self,data):
if self.metadata is not None:
return "%s::%s" % (self.metadata, data)
elif re.search("GET", data) != None:
data = re.split('[ ,]', data)
csv = data[0] + ' ' + data[1] + ',' + data[7] + ',' + data[9]
return csv
''' batch not currently working
def flush_messages(self):
global pending_messages
print "flushing %d messages " % len(pending_messages)
self.producer.send_messages(self.topic,pending_messages)
pending_messages = []
def send_to_kafka(self,message_text):
global pending_messages
pending_messages.append(message_text)
if len(pending_messages) == self.batch_size:
self.flush_messages(self.producer)
'''
def log_lines_generator(self, logfile, delay_between_iterations=None):
global should_stop
cmd = ['tail', '-n', '0', '-F']
if delay_between_iterations is not None:
cmd.append('-s')
cmd.append(delay_between_iterations)
cmd.append(logfile)
process = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=None)
while not should_stop:
line = process.stdout.readline().strip()
yield line
def run(self):
self.client = KafkaClient(self.host + ':' + str(self.port))
self.producer = SimpleProducer(self.client)
try:
for line in self.log_lines_generator(self.logfile):
msg = self.create_message_data(line)
self.producer.send_messages(self.topic, msg)
except KeyboardInterrupt,e:
pass
def emit(self, container, event):
try:
producer = self.producers[event.evt_type]
except KeyError:
producer = SimpleProducer(self.client, event.evt_type)
self.producers[event.evt_type] = producer
producer.send_messages(json.dumps(event.serialize()))
def output_kafka(graph_db, registry,
kafka_url=None):
ldict = {"step": MODULEFILE + "/" + inspect.stack()[0][3],
"hostname": platform.node().split(".")[0]}
l = logging.LoggerAdapter(common.fetch_lg(), ldict)
kafka_topic = "cs"
if kafka_url is None:
kafka_url = registry.get_config("kafka_url",
"localhost:9092")
else:
l.info("Updating registry with kafka_url: {}".format(kafka_url))
registry.put_config("kafka_url",
kafka_url)
(nodes, rels) = out.output_json(graph_db, None, None, as_list=True)
l.info("Connecting to kafka_url {}".format(kafka_url))
kafka = KafkaClient(kafka_url)
# To send messages asynchronously
producer = SimpleProducer(kafka)
l.info("Sending nodes to kafka {}/{}".format(kafka_url, kafka_topic))
for n in nodes:
producer.send_messages(kafka_topic, n)
l.info("Sending rels to kafka {}/{}".format(kafka_url, kafka_topic))
for n in rels:
producer.send_messages(kafka_topic, n)
kafka.close()
def main():
happy_log_probs, sad_log_probs = readSentimentList(
'twitter_sentiment_list.csv')
consumer = KafkaConsumer("tweets", bootstrap_servers=["localhost:9092"],
auto_offset_reset='smallest')
kafka = KafkaClient("localhost:9092")
producer = SimpleProducer(kafka)
topic = 'hashtag_sentiment'
positive_tags = Counter()
negative_tags = Counter()
while True:
for message in consumer.fetch_messages():
txt = message.value
txt = re.sub(r'[^\x00-\x7F]', ' ', txt)
hashtags, sentiment = classifySentiment(
txt, happy_log_probs, sad_log_probs)
for hashtag in hashtags:
if sentiment > 0:
positive_tags[hashtag] += 1
else:
negative_tags[hashtag] += 1
results = {}
for key, val in positive_tags.most_common(20):
results[key] = val
producer.send_messages(topic, json.dumps(results))
time.sleep(10)
class KafkaConnector(object):
def __init__(self, host_name, host_port):
self.client = KafkaClient(host_name + ":" + host_port)
self.producer = SimpleProducer(self.client)
def create_topic(self, topic_name):
topic_exists = self.client.has_metadata_for_topic(topic_name)
if not topic_exists:
self.client.ensure_topic_exists(topic_name)
def send_message(self, topic_name, message):
self.producer.send_messages(topic_name, message)
def register_consumer(self, callback, parse_json, topic_group, topic_name):
consumer = SimpleConsumer(self.client, topic_group, topic_name)
consumer_thread = ConsumerThread(consumer, callback, parse_json)
consumer_thread.start()
def blocking_consumer(self, message_consume_function, parse_json, topic_group, topic_name):
print "starting blocking consumer with topic group %s and topic name %s" % (topic_group, topic_name)
consumer = SimpleConsumer(self.client, topic_group, topic_name)
consumer.seek(0,2)
for message in consumer:
message = parse_json(message)
print "=============" + str(message) + "============"
message_consume_function(message)
print "called message consume function"
def main():
client = KafkaClient("localhost:9092")
producer = SimpleProducer(client)
for i in range(5):
producer.send_messages('mytopic', "This is my test message, number {0}".format(i))
time.sleep(1)
class Producer(threading.Thread):
daemon = True
def __init__(self, broker, topic, directory):
threading.Thread.__init__(self)
self.client = KafkaClient(broker)
self.producer = SimpleProducer(self.client)
self.message_dir = directory
self.topic = topic
def send_file(self, filename):
print "Send:", filename
with file(filename) as f:
file_contents = f.read()
self.producer.send_messages(self.topic, file_contents)
def run(self):
onlyfiles = [
f for f in listdir(self.message_dir)
if isfile(join(self.message_dir, f))
]
for f in onlyfiles:
filename = join(self.message_dir, f)
self.send_file(filename)
time.sleep(10)
class KafkaMetrics(object):
def __init__(self, broker_list, kafka_topic):
logging.basicConfig(level=logging.INFO)
self.log = logging.getLogger('druid-kafka-metrics')
self.log.info("Kafka (brokers=%s, topic=%s)" %
(broker_list, kafka_topic))
client = KafkaClient(broker_list)
self.producer = SimpleProducer(client)
self.msg_count = 0
self.kafka_topic = kafka_topic
@cherrypy.expose
@cherrypy.tools.json_out()
@cherrypy.tools.json_in()
def metrics(self):
messages = cherrypy.request.json
for message in messages:
self.msg_count += 1
self.log.debug("%s - %s" % (self.msg_count, str(message)))
self.producer.send_messages(self.kafka_topic, json.dumps(message))
if self.msg_count % 100 == 0:
self.log.info("%s messages processed." % (self.msg_count, ))
return "{'code':200}"
def main():
global freq_array
client = KafkaClient('ip-172-31-28-55.ec2.internal:6667')
producer = SimpleProducer(client)
fft_size = 1000
fs = 92
freq_array = np.array((1 * fs / fft_size))
for i in range(2, int(fft_size / 2)):
freq_i = np.array((i * fs / fft_size))
freq_array = np.vstack((freq_array, freq_i))
with open('xfourmag.csv', 'rt') as f:
print('opening csv')
reader = csv.reader(f)
row = next(reader)
#global mags
mags = np.array(row)
for row in reader:
#mags += row
mags = np.vstack((mags, row))
#print(mags)
#print(freq_array)
json_data = {
'time': int(time.time()),
'fft': np.hstack((freq_array[0:31], mags[0:31])).tolist(),
'sensor_id': '1',
'reading_type': '0'
}
print('sending data...')
producer.send_messages('shm', (json.dumps(json_data)).encode('utf-8'))
print('data sent! :)')
def forwarder(self):
client = KafkaClient(hosts(self.server_list, self.kafka_port))
client.ensure_topic_exists(self.topic_name)
producer = SimpleProducer(client, batch_send=False)
print producer
for i in xrange(1, 100):
with open(self.csvfile, 'r') as FR:
fields = next(FR).strip().split('\t')
print fields
for cnc_log in FR:
values = cnc_log.strip().split('\t')
zipped = dict(zip(fields, values))
zipped['lower_bound'] = float(zipped['lower_bound'])
zipped['upper_bound'] = float(zipped['upper_bound'])
zipped['temperature'] = float(zipped['temperature'])
zipped['no'] = int(zipped['no'])
print json.dumps(zipped, sort_keys=True, indent=4)
# prob = 0.8
# y = lambda x, prob: '<span style="background-color:#bd362f; color:white">FAIL</span>' if randint(0,x) > x*prob else 'PASS'
# cnc_log = (datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')+"\t"+y(10,0.8)+'\t'+cnc_log.strip()).split('\t')
# zipped = dict(zip(fields,cnc_log))
# node = zipped
sleep_sec = random.uniform(0, 3) * 5
time.sleep(sleep_sec)
producer.send_messages(self.topic_name, json.dumps(zipped))
def kafkahighlevelproducer(kafka_conn, schema, bytes):
"""
kafka High level API
"""
print "SimpleProducer start."
writer = cStringIO.StringIO()
encoder = avro.io.BinaryEncoder(writer)
#JSONEncoder is not implemented ..
#encoder = avro.io.JSONEncoder(writer)
datum_writer = avro.io.DatumWriter(schema)
producer = SimpleProducer(kafka_conn)
for topic in ["DUMMY_LOG"]:
writer.truncate(0)
datum_writer.write({"id": 100L, "logTime": 20140401L, "muchoStuff": {"test": "test1value"}}, encoder)
bytes = writer.getvalue()
producer.send_messages(topic, bytes)
writer.truncate(0)
datum_writer.write({"id": 101L, "logTime": 20140402L, "muchoStuff": {"test": "test2value"}}, encoder)
bytes = writer.getvalue()
producer.send_messages(topic, bytes)
writer.close()
print "SimpleProducer done."
def main():
global freq_array
client = KafkaClient('ip-172-31-28-55.ec2.internal:6667')
producer = SimpleProducer(client)
fft_size=1000
fs=92
freq_array=np.array((1*fs/fft_size))
for i in range(2,int(fft_size/2)):
freq_i=np.array((i*fs/fft_size))
freq_array=np.vstack((freq_array,freq_i))
with open('xfourmag.csv', 'rt') as f:
print('opening csv')
reader=csv.reader(f)
row = next(reader)
#global mags
mags = np.array(row)
for row in reader:
#mags += row
mags = np.vstack((mags,row))
#print(mags)
#print(freq_array)
json_data = {'time': int(time.time()), 'fft': np.hstack((freq_array[0:31],mags[0:31])).tolist(), 'sensor_id': '1', 'reading_type': '0'}
print('sending data...')
producer.send_messages('shm', (json.dumps(json_data)).encode('utf-8'))
print('data sent! :)')
def seed_kafka(kafka_hosts=None, topic_name=None, num_emails=100000):
"""Seed the local Kafka cluster's "dumpmon" topic with sample email data."""
topic_name = topic_name or "dumpmon"
kafka_hosts = kafka_hosts or "127.0.0.1:9092"
kafka = KafkaClient(kafka_hosts)
producer = SimpleProducer(kafka)
# producer = SimpleProducer(kafka, batch_send=True, batch_send_every_n=1000,
# batch_send_every_t=5)
print("Seeding Kafka ({}) topic '{}' with {:,} fake emails."
.format(kafka_hosts, topic_name, num_emails))
emails = random_email_generator()
for i in range(num_emails):
email = json.dumps(next(emails)).encode("utf-8", "ignore")
producer.send_messages(topic_name, email)
print("Done.")
class Producer(object):
def __init__(self, addr):
self.client = KafkaClient(addr)
self.producer = SimpleProducer(self.client)
def produce_msgs(self):
msg_cnt = 0
while True:
timestamp=rutil.randomDate("2015,10,01,00,00,00", "2015,10,06,23,59,00", random.random())
phone = fake.phone_number()
latdep,longdep=rutil.randomSingleLatLon(LATDEP, LONGDEP, RADIUS)
latarr,longarr=rutil.randomSingleLatLon(LATARR, LONGARR, RADIUS)
message_info = rutil.create_json_message(phone=phone,
latdep=latdep,
longdep=longdep,
latarr=latarr,
longarr=longarr,
timestamp=timestamp,
dr_flag=random.choice([0, 1]),
user_id=random.choice(range(NUM_USERS)),
message_id=msg_cnt)
print 'message ' +str(msg_cnt) +' ---------- '+ message_info
self.producer.send_messages('RideRequests',message_info)
msg_cnt += 1
def forwarder(self):
client = KafkaClient(hosts(self.server_list, self.kafka_port))
client.ensure_topic_exists(self.topic_name)
producer = SimpleProducer(client, batch_send=False)
print producer
no = 1
for i in xrange(1,10000):
with open(self.csvfile, 'r') as FR:
first_line = next(FR)
print first_line
fields = first_line.lstrip().rstrip().split('\t')
print fields
for cnc_log in FR:
print cnc_log
values = cnc_log.strip().split('\t')
zipped = dict(zip(fields,values))
zipped['lower_bound'] = float(zipped['lower_bound'])
zipped['upper_bound'] = float(zipped['upper_bound'])
zipped['spindle'] = float(zipped['spindle'])
# zipped['no'] = int(zipped['no'])
zipped['no'] = no
zipped['tool_no'] = int(zipped['tool_no'])
# zipped['tool_no'] = i
print json.dumps(zipped,sort_keys=True,indent=4)
sleep_sec = 1
time.sleep(sleep_sec)
producer.send_messages(self.topic_name, json.dumps(zipped))
no = no +1
def __init__(self):
comlog.init_logger("./../log/vdata.log")
self.ub_conf = UbConfig("./../conf/log_tail.conf")
self._conf_info = self.ub_conf.get_conf_info()
self._file_path = self._conf_info["[LOG_FILE_CONF]"]["file_path"]
self._file_name_pattern = self._conf_info["[LOG_FILE_CONF]"][
"file_name_pattern"]
self._log_max_length = int(
self._conf_info["[LOG_FILE_CONF]"]["log_max_length"])
self._batch_flush_counter = int(
self._conf_info["[LOG_FILE_CONF]"]["batch_flush_counter"])
self._topic_name = self._conf_info["[KAFKA]"]["topic_name"]
self._interval_time = self._conf_info["[TIME_INTERVAL]"]["interval"]
self.init_data_file()
self.s = sched.scheduler(time.time, time.sleep)
if self._conf_info["[KAFKA]"].has_key('broker_list'):
self.broker_list = self._conf_info["[KAFKA]"]["broker_list"]
elif self._conf_info["[KAFKA]"].has_key('zookeeper'):
self.broker_list = ','.join(
self.get_broker_list(self._conf_info["[KAFKA]"]['zookeeper']))
else:
raise ValueError, " zookeeper and broker_list are both null in config file"
self.client = KafkaClient(self.broker_list)
self.producer = SimpleProducer(self.client)
#tags
self.tags = {}
for key in self._conf_info["[TAGS]"]:
self.tags[key] = self._conf_info["[TAGS]"][key]
def initialize(self, storm_conf, context):
#TODO debug storm_conf, context
self.client = InfluxDBClient("streamparse-box", 8086, "root", "root",
"prices_development")
kafka = KafkaClient(KAFKA_HOST)
self.queue = SimpleProducer(
kafka, batch_send=True, batch_send_every_n=1,
batch_send_every_t=1) # batching config can be tuned
def run(self):
client = KafkaClient("localhost:9092")
producer = SimpleProducer(client)
self.sent = 0
while True:
producer.send_messages('my-topic', self.big_msg)
self.sent += 1
def get_producer(self):
"""
:return: SimpleProducer
"""
if not self.producer:
self.get_client()
self.producer = SimpleProducer(self.client)
return self.producer
def __init__(self, topic, hosts=None, log_level=logging.WARNING):
hosts = hosts or "localhost:9092"
self.topic = "{}_{}".format("kafque", topic)
self.client = KafkaClient(hosts)
self.client.ensure_topic_exists(str(self.topic))
self.producer = SimpleProducer(
self.client, req_acks=SimpleProducer.ACK_AFTER_CLUSTER_COMMIT)
self.logger = setup_logger(__name__, level=log_level)
def main():
client = KafkaClient("localhost:9092")
producer = SimpleProducer(client)
for i in range(5):
producer.send_messages('mytopic',
"This is my test message, number {0}".format(i))
time.sleep(1)
def run(self):
avro_serde = AvroSerDe(AVRO_SCHEMA_STRING)
client = KafkaClient('localhost:9092')
producer = SimpleProducer(client)
while True:
raw_bytes = avro_serde.obj_to_bytes(get_login_event())
producer.send_messages(KAFKA_TOPIC, raw_bytes)
time.sleep(1)
def __init__(self, settings):
super(ImageConvertProcess, self).__init__(settings)
self.kafka_client = KafkaClient(self.settings.get("KAFKA_HOSTS"))
self.kafka_client.ensure_topic_exists(self.topic_name)
self.producer = SimpleProducer(self.kafka_client)
#self.lock = RLock()
self.IC = ImageConvert(settings)
self.IC.set_logger(self.logger)
def run(self):
client = KafkaClient("localhost:9092")
producer = SimpleProducer(client)
self.sent = 0
while True:
producer.send_messages("my-topic", self.big_msg)
self.sent += 1
def sendTokafka(partitionData):
record = partitionData
Records = []
new_record = {'Data':record}
Records.append(new_record)
kafka1 = KafkaClient("localhost:9092")
producer1 = SimpleProducer(kafka1)
producer1.send_messages("too",bytes(Records))
return 0
def run(self):
client = KafkaClient("localhost:9092")
producer = SimpleProducer(client)
self.sent = 0
i=1
while i<10:
producer.send_messages('my-topic', self.big_msg)
self.sent += 1
i +=1
def run(self):
client = KafkaClient(self.bootstrap_server)
producer = SimpleProducer(client)
i = 0
while True:
d = datetime.now()
producer.send_messages(self.topic, ("%s:%s" % (i, d)).encode('utf-8'))
i += 1
time.sleep(2)
def __init__(self, addr):
self.client = KafkaClient(addr)
self.producer = SimpleProducer(self.client)
with open('AdReports/FB_Report/accessToken.txt', 'rb') as f:
self.access_token_list = pickle.load(f)
with open('AdReports/FB_Report/accountID.txt', 'rb') as d:
self.account_id_list = pickle.load(d)
def run(self):
client = KafkaClient("localhost:9092")
producer = SimpleProducer(client)
while True:
producer.send_messages('my-topic', "test")
producer.send_messages('my-topic', "\xc2Hola, mundo!")
time.sleep(1)
def run(self):
client = KafkaClient("10.206.216.13:19092,10.206.212.14:19092,10.206.209.25:19092")
producer = SimpleProducer(client,async=True)
while True:
producer.send_messages('guantest', "test")
producer.send_messages('guantest', "test")
time.sleep(1)
def __init__(self, broker_list, kafka_topic):
logging.basicConfig(level=logging.INFO)
self.log = logging.getLogger('druid-kafka-metrics')
self.log.info("Kafka (brokers=%s, topic=%s)" %
(broker_list, kafka_topic))
client = KafkaClient(broker_list)
self.producer = SimpleProducer(client)
self.msg_count = 0
self.kafka_topic = kafka_topic
def __init__(self, broker):
try:
self.client = KafkaClient(broker)
self.prod = SimpleProducer(self.client)
except KafkaUnavailableError:
log.critical("\nCluster Unavailable %s : Check broker string\n",
broker)
raise
except:
raise
def run(self):
client = KafkaClient("localhost:6379")
producer = SimpleProducer(client)
i=0
while (i<5):
a=askOrder()
producer.send_messages(a.product, str(a.mode) + "," +str(a.volume) + "," + str(a.price))
time.sleep(1)
i=i+1
def _feed(settings_file, json_item):
settings = importlib.import_module(settings_file[:-3])
kafka_conn = KafkaClient(settings.KAFKA_HOSTS)
topic = settings.KAFKA_INCOMING_TOPIC
producer = SimpleProducer(kafka_conn)
print "=> feeding JSON request into {0}...".format(topic)
print json.dumps(json_item, indent=4)
kafka_conn.ensure_topic_exists(topic)
producer.send_messages(topic, json.dumps(json_item))
print "=> done feeding request."
def run(self):
client = KafkaClient("172.17.8.101:9092")
producer = SimpleProducer(client)
global_counter = 0
while True:
dict = {'P_1': rand(), 'P_2': rand(), 'P_3': rand(), 'Q_1': rand(), 'Q_2': rand(), 'Q_3': rand(), 'timestamp': int(time.time())*1000, 'id': str(uuid.uuid4())};
producer.send_messages('topic', json.dumps(dict))
global_counter += 1
print global_counter
def __init__(self, factory, destination, message_size):
self.factory = factory
self.destination = destination
self.producer = SimpleProducer(factory, self.destination)
self.rate = PerfRate()
self.message_size = message_size
self.message = ""
for i in range(message_size):
self.message += "a"
threading.Thread.__init__(self)
def run(self):
client = KafkaClient(settings.KAFKA_SERVER)
producer = SimpleProducer(client)
while True:
producer.send_messages(
'heartbeats',
"""{"id":"yes-is-a-fake-uuide", "status": 200, "serviceName":"chewit_cam" }"""
)
time.sleep(5)
def run(self):
client = KafkaClient("10.206.216.13:19092,10.206.212.14:19092,10.206.209.25:1909")
producer = SimpleProducer(client)
#producer = KeyedProducer(client,partitioner=HashedPartitioner)
while True:
producer.send_messages('jiketest', "test")
producer.send_messages('jiketest', "test")
time.sleep(1)
def run(self):
client = KafkaClient(
"10.206.216.13:19092,10.206.212.14:19092,10.206.209.25:1909")
producer = SimpleProducer(client)
#producer = KeyedProducer(client,partitioner=HashedPartitioner)
while True:
producer.send_messages('guantest', "test")
producer.send_messages('guantest', "test")
time.sleep(1)
def main():
kafka = KafkaClient("localhost:9092")
producer = SimpleProducer(kafka)
consumer = SimpleConsumer(kafka, "my-group", "activity.stream", max_buffer_size=None)
producer.send_messages("activity.stream", "some message test")
for message in consumer:
print(message)
kafka.close()
def run(self):
self.client = KafkaClient(settings.KAFKA_SERVER)
self.producer = SimpleProducer(self.client)
while True:
data = {
"id": str(uuid.uuid4()),
"status": 200,
"serviceName": settings.RIDE
}
self.producer.send_messages("heartbeats", json.dumps(data))
time.sleep(5)
class Producer(object):
"""Kafka producer class with functions to send messages.
Messages are sent to a single kafka topic "messages" as a json formatted
string
Attributes:
client: string representing IP:port of the kafka broker
producer: Producer object using the previously specified kafka client
county_state_list: a list of lists containing the counties and their
associated state abbreviation
"""
def __init__(self, addr):
"""Initialize Producer with address of the kafka broker ip address."""
self.client = KafkaClient(addr)
self.producer = SimpleProducer(self.client)
self.county_state_list = IngUt.parse_county_list('ingestion/county_list.txt')
def sim_msg_stream(self, sleep_time=0.25):
"""Sends a stream of messages to the Kafka topic "messages".
Args:
sleep_time: float number in seconds representing the rate messages
should be sent to the Kafka topic
Returns:
None
"""
msg_cnt = 0
while True:
print len(self.county_state_list)
county, state = IngUt.select_random_county(self.county_state_list)
timestamp = list(time.localtime()[0:6])
message_info = IngUt.create_json_message(county=county,
state=state,
rank=0,
timestamp=timestamp,
creator_id=random.randint(0, NUM_USERS-1),
sender_id=random.randint(0, NUM_USERS-1),
message_id=msg_cnt,
message=fake.text())
self.producer.send_messages('messages', message_info)
print timestamp
if sleep_time != 0:
time.sleep(sleep_time)
msg_cnt += 1
def feed(self, json_item):
'''
Feeds a json item into the Kafka topic
@param json_item: The loaded json object
'''
topic = self.settings.KAFKA_INCOMING_TOPIC
producer = SimpleProducer(self.kafka_conn)
print "=> feeding JSON request into {0}...".format(topic)
print json.dumps(json_item, indent=4)
self.kafka_conn.ensure_topic_exists(topic)
producer.send_messages(topic, json.dumps(json_item))
print "=> done feeding request."
def feed(self, json_item):
'''
Feeds a json item into the Kafka topic
@param json_item: The loaded json object
'''
topic = self.settings.KAFKA_INCOMING_TOPIC
producer = SimpleProducer(self.kafka_conn)
print "=> feeding JSON request into {0}...".format(topic)
print json.dumps(json_item, indent=4)
self.kafka_conn.ensure_topic_exists(topic)
producer.send_messages(topic, json.dumps(json_item))
print "=> done feeding request."
def publish_training_data(kafka_server, kafka_topic, num_to_send, sleep_every_N, target_a, target_b):
kafka = KafkaClient(kafka_server)
producer = SimpleProducer(kafka)
for c in range(num_to_send):
x, y = util.generate_linear_x_y_data(1, target_a, target_b, c)
# print "%d %.20f %.20f" % (int(round(time.time() * 1000)), x[0], y[0])
producer.send_messages(
kafka_topic,
"%d %.5f %.5f %.20f %.20f" % (util.now_millis(), target_a, target_b, x[0], y[0]))
if (c + 1) % sleep_every_N == 0:
time.sleep(1)
class Producer():
def __init__(self, server_list, kafka_port, topic_name):
self.server_list = server_list
self.kafka_port = kafka_port
self.topic_name = topic_name
self.client = KafkaClient(hosts(self.server_list, self.kafka_port))
self.producer = SimpleProducer(self.client, batch_send=False)
def ensure_topic_exists(self):
self.client.ensure_topic_exists(self.topic_name)
def forwarder(self, message):
self.producer.send_messages(self.topic_name, message)
def executions(test=False, kafka_hosts=None, topic_name='executions', num_fills=1):
"""Seed random executions"""
kafka_hosts = kafka_hosts or KAFKA_HOST
kafka = KafkaClient(kafka_hosts)
producer = SimpleProducer(kafka, batch_send=True, batch_send_every_n=1,
batch_send_every_t=1)
fills = random_execution_generator()
for i in range(num_fills):
fill = json.dumps(next(fills)).encode("utf-8","ignore")
print(fill)
if test is False:
producer.send_messages(topic_name, fill)
print("Done.")
def queue_fills(test=False, kafka_hosts=None, topic_name='fills', num_fills=1):
"""Seed the local Kafka cluster's "fills" topic with sample fill data."""
kafka_hosts = kafka_hosts or KAFKA_HOST
kafka = KafkaClient(kafka_hosts)
producer = SimpleProducer(kafka, batch_send=True, batch_send_every_n=1,
batch_send_every_t=1)
fills = random_fill_generator()
for i in range(num_fills):
fill = json.dumps(next(fills)).encode("utf-8","ignore")
print(fill)
if test is False:
producer.send_messages(topic_name, fill)
print("Done.")
class KafkaProducer(KafkaBase, TerminatorBlock):
""" A block for producing Kafka messages """
version = VersionProperty("1.0.1")
def __init__(self):
super().__init__()
self._producer = None
def configure(self, context):
super().configure(context)
self._connect()
def stop(self):
self._disconnect()
super().stop()
def process_signals(self, signals, input_id='default'):
msgs = []
for signal in signals:
if self.connected:
try:
if type(signal) is not bytes:
signal = pickle.dumps(signal)
except:
self.logger.exception(
"Signal: {0} could not be serialized".format(signal))
return
msgs.append(signal)
else:
return
try:
if self.connected:
self._producer.send_messages(self._encoded_topic, *msgs)
except:
self.logger.exception("Failure sending signal")
def _connect(self):
super()._connect()
self._producer = SimpleProducer(self._kafka)
def _disconnect(self):
if self._producer:
self._producer.stop()
self._producer = None
super()._disconnect()
@property
def connected(self):
return super().connected and self._producer
def send_msg(msgs):
cli = KafkaClient("localhost:9092")
producer = SimpleProducer(cli)
if isinstance(msgs, list):
content = [(json.dumps(msg) if isinstance(msg, dict) else msg) for msg in msgs]
else:
content = [msgs]
try:
resp = producer.send_messages("tp_test1", *content)
print resp
except Exception:
print traceback.format_exc()
finally:
cli.close()
class OSCollector(threading.Thread):
path = None
kafka = None
producer = None
kafkaHost = None #'9.110.95.141:9092'
hostname = None
def __init__(self, kafkaHost):
if kafkaHost is None:
raise SyntaxError('Unknown kafka server')
self.kafkaHost = kafkaHost
threading.Thread.__init__(self)
p = subprocess.Popen('hostname -f',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for line in p.stdout.readlines():
self.hostname = line.lstrip().rstrip()
break
self.path = '/root/x.json'
self.kafka = KafkaClient(self.kafkaHost)
self.producer = SimpleProducer(self.kafka)
def run(self):
while True:
cpu = self.get_cpu_usage()
memory = self.get_mem_usage()
message = {}
message["node"] = self.hostname
systemObject = {}
systemObject["cpu"] = cpu
systemObject["memory"] = memory
message["system"] = systemObject
try:
print str(message)
self.producer.send_messages("collector",
str(json.dumps(message)))
except Exception, e:
print str(e)
time.sleep(10)
def queue_fills(test=False, kafka_hosts=None, topic_name='fills', num_fills=1):
"""Seed the local Kafka cluster's "fills" topic with sample fill data."""
kafka_hosts = kafka_hosts or KAFKA_HOST
kafka = KafkaClient(kafka_hosts)
producer = SimpleProducer(kafka,
batch_send=True,
batch_send_every_n=1,
batch_send_every_t=1)
fills = random_fill_generator()
for i in range(num_fills):
fill = json.dumps(next(fills)).encode("utf-8", "ignore")
print(fill)
if test is False:
producer.send_messages(topic_name, fill)
print("Done.")
