def sendTwitterTweetsToKafka(body):
producer = KafkaProducer(bootstrap_servers="10.0.0.9:9092, 10.0.0.12:9092, 10.0.0.6:9092, 10.0.0.14:9092")
resp = producer.send('marsstreaming', body)
producer.flush()
return resp
from kafka import KafkaProducer
import random
import json
from json import dumps
from datetime import datetime
bootstrap_servers = ['localhost:9091', 'localhost:9092', 'localhost:9093']
# topicName = 'my-topic-three'
topicName = 'my-topic-msg02-sem-flush'
# producer = KafkaProducer(bootstrap_servers = bootstrap_servers)
# configuração do kafka
producer = KafkaProducer(bootstrap_servers=bootstrap_servers,
value_serializer=lambda x: dumps(x).encode('utf-8'))
# producer.send(topicName, b'Hello World!')
# Call the producer.send method with a producer-record
print("Ctrl+c to Stop")
total = 0
while True:
total += 1
# print(random.randint(1,999))
# print(total)
# producer.send('topicName', str(random.randint(1,999)).encode())
data_e_hora_completa = datetime.now()
data_string = data_e_hora_completa.strftime('%Y-%m-%d %H:%M:%S')
msg = 'Olá pessoas, agora sem flush e com o consumer desligado!!!'
dados = {"msg": str(msg), "horario": data_string, "total": total}
# producer.send(topicName, b'Hello World!')
print(dados)
from kafka import KafkaProducer
import threading
import metrics
import json
#JoeBiden id: 939091
#realDonaldTrump id: 25073877
# Twitter API access keys
access_token = "1323451448781238274-ARnzAE9Jr9T4KjiruQSqIJM4WnAiCk"
access_token_secret = "KfMietdsdxqRqKiEVVEJdKypfGbrWjFaE9Y92Y1ij4ilk"
consumer_key = "doz9ZltVGwMmlg2PTptGcoMLd"
consumer_secret = "0BrrMlwj443opgxpjkowqJ18vqrwLnHUaY1eooYTvL4xZmtfFJ"
# Topic and producer
topic = "default_topic"
producer = KafkaProducer(bootstrap_servers='localhost:9092')
class TwitterAuthenticator():
def authenticate_twitter_app(self):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
return auth
class TwitterStreamProducer():
def __init__(self, topic_name):
global topic
topic = topic_name
self.twitter_authenticator = TwitterAuthenticator()
self.metrics = metrics.Metrics(producer=producer)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jan 22 16:44:22 2021
@author: Saba Kiriako
"""
import json
from kafka import KafkaConsumer, KafkaProducer
stations = {}
consumer = KafkaConsumer("velib-stations", bootstrap_servers='localhost:9092')
host = "9092"
producer = KafkaProducer(bootstrap_servers="localhost:" + str(host))
topic_name = "stations-status"
statuses = {}
for message in consumer:
i = 0
stations = json.loads(message.value.decode())
for station in stations:
key = "{}-{}-{}".format(station["number"], station["name"],
station["contract_name"])
status = station["available_bikes"]
if key not in statuses.keys():
statuses[key] = status
else:
if status != statuses[key]:
print("Current Status :" + str(status) + " Previous Status :" +
str(statuses[key]))
from kafka import KafkaProducer
from kafka.errors import KafkaError
from setting import log
producer = KafkaProducer(bootstrap_servers=['broker1:1234'])
# Asynchronous by default
future = producer.send('my-topic', b'raw_bytes')
# Block for 'synchronous' sends
try:
record_metadata = future.get(timeout=10)
except KafkaError:
# Decide what to do if produce request failed...
log.exception()
pass
# Successful result returns assigned partition and offset
print(record_metadata.topic)
print(record_metadata.partition)
print(record_metadata.offset)
# produce keyed messages to enable hashed partitioning
producer.send('my-topic', key=b'foo', value=b'bar')
# encode objects via msgpack
producer = KafkaProducer(value_serializer=msgpack.dumps)
producer.send('msgpack-topic', {'key': 'value'})
# produce json messages
producer = KafkaProducer(
from kafka import KafkaProducer, KafkaConsumer
from kafka import KafkaClient, SimpleConsumer
import time
import sys
from app import app
from flask import Flask, render_template, request, redirect, Response
import random, json
SEND_TIME = None
print("At top of function \n\n")
RECEIVE_TIME = None
bs = ['54.218.73.149:9092','50.112.197.74:9092','34.222.135.111:9092']
PRODUCER = KafkaProducer(bootstrap_servers=bs)
CLIENT = KafkaClient(bs)
@app.route('/')
def home():
return render_template('setuser.html')
@app.route('/<user>')
def serve_user(user):
consumer = SimpleConsumer(CLIENT, 'testing', 'user{}_sess{}'.format(user,user))
msg = None
msg = consumer.get_message()
RECEIVE_TIME = time.time()
color='yellow'
from xmlrpc.server import SimpleXMLRPCServer
import requests
from time import sleep
from json import dumps
from kafka import KafkaProducer
sleep(20)
producer = KafkaProducer(bootstrap_servers=['localhost:9092'],
value_serializer=lambda x: dumps(x).encode('utf-8'))
# for e in range(1000):
# data = {'number' : e}
# producer.send('numtest', value=data)
# sleep(5)
pollServer = SimpleXMLRPCServer(('0.0.0.0', 9000),
logRequests=True,
allow_none=True)
# ROCKETS_STATES_BASE_URL = "http://localhost:5000"
# ELON_URL = "http://localhost:8000/"
# TORY_URL = "http://localhost:3000/"
def getResponsesPoll(siteName, rocketName):
print("-----------------------------------")
print(siteName + "\n" + rocketName)
data = {'siteName': siteName, 'rocketName': rocketName}
producer.send('Pollrequesttopic', value=data)
def __init__(self):
# localhost:9092 = Default Zookeeper Producer Host and Port Adresses
self.producer = KafkaProducer(bootstrap_servers=['localhost:9092'])
stream.map(pair).reduceByKey(lambda a, b: (a[0] + b[0], a[1] + b[1])).map(
lambda (k, v): (k, v[0] / v[1])).foreachRDD(send_to_kafka)
if __name__ == '__main__':
if len(sys.argv) != 4:
print("Usage: stream-process.py [topic] [target-topic] [broker-list]")
exit(1)
# - create SparkContext and StreamingContext
sc = SparkContext("local[2]", "StockAveragePrice")
sc.setLogLevel('INFO')
ssc = StreamingContext(sc, 5)
topic, target_topic, brokers = sys.argv[1:]
# - instantiate a kafka stream for processing
directKafkaStream = KafkaUtils.createDirectStream(
ssc, [topic], {'metadata.broker.list': brokers})
process_stream(directKafkaStream)
# - instantiate a simple kafka producer
kafka_producer = KafkaProducer(bootstrap_servers=brokers)
# - setup proper shutdown hook
atexit.register(shutdown_hook, kafka_producer)
ssc.start()
ssc.awaitTermination()
from kafka import KafkaProducer
import sys
import time
import datetime
import pandas as pd
import json
producer = KafkaProducer(bootstrap_servers=['kafka1:9092'])
topic_name = "streamKafka"
print("sending messages to topic:" + str(topic_name))
#Compatible with logstash
for chunck_df in pd.read_csv("../Data/miniTraffic.csv", chunksize=100):
for index, point in chunck_df.iterrows():
# Los puntos no funcionan bien en las columnas de spark
#point.index = [x.replace(".","_") for x in point.index]
point['location'] = [
point['coordinates_long'], point['coordinates_lat']
]
point['dateSend'] = datetime.datetime.now()
msg = point.to_json()
msg2 = json.loads(msg, encoding='utf-8')
msgJson = json.dumps(msg2)
producer.send(str(topic_name), bytes(msgJson + '\n', 'utf-8'))
time.sleep(0.0016)
producer.close()
from kafka import KafkaConsumer, KafkaProducer
import time
import json
from MyConsumer import *
from log import KafkaLog
from conf.getConf import *
consumer = KafkaConsumer('testyg',
group_id="test_group_1",
bootstrap_servers=['10.4.10.239:9092'],
enable_auto_commit=False)
producer = KafkaProducer(bootstrap_servers='10.4.10.239:9092') # 连接kafka
clintInf = ClientInf(0, 18)
log = KafkaLog()
localHost = getLocalhost()
pwd = os.path.dirname(__file__)
with open(pwd + '/imgs/label/imagenetLabel.json') as f:
labels = json.load(f)
def class_id_to_label(i):
return labels[i]
def dataConsume():
cnt = 0
for msg in consumer:
# recv = "%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value)
#
# print(recv)
trace[AUX_VALID_TRACE] = 1
return trace
def print_active_connections():
print("############################")
print(active_connections)
def get_active_connections():
return active_connections
consumer = KafkaConsumer('sniffer', bootstrap_servers=['10.40.39.22:1025'])
producer = KafkaProducer(bootstrap_servers='10.40.39.22:1025')
print('Started Consumer')
for message in consumer:
package_info = message.value.decode('utf-8').replace("[", "").replace(
"]", "").replace('"', "").replace("'", "").split(' ')
parsed_trace, trace, protocol, total_active_connections = update_active_connections(
package_info)
if parsed_trace is not None:
new_parsed_trace = np.array([])
for i in range(len(parsed_trace)):
new_parsed_trace = np.append(new_parsed_trace,
str(parsed_trace[i]))
new_parsed_trace = np.append(new_parsed_trace, str(protocol))
producer.send('connections', np.array_str(new_parsed_trace).encode())
print(new_parsed_trace)
'https://polyglot-academy-pub.nyc3.digitaloceanspaces.com/liquor-ml')
bucket_source_name = bucket_source.split('/')[-1]
bucket_destination = os.getenv('BUCKET_BASE_NAME', 'liquor-images')
# Helper database
db_user = os.getenv('DATABASE_USER', 'liquorlab')
db_password = os.getenv('DATABASE_PASSWORD', 'liquorlab')
db_host = os.getenv('DATABASE_HOST', 'liquorlabdb')
db_db = os.getenv('DATABASE_DB', 'liquorlabdb')
# Delay between images
seconds_wait = float(os.getenv('SECONDS_WAIT', 2))
# Kakfa producer
producer = KafkaProducer(
bootstrap_servers='my-cluster-kafka-bootstrap:9092',
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
########
# Code #
########
def copy_file(source, image_key, destination, image_name):
"""Copies an object from a URL source to a destination bucket."""
image_url = source + '/' + image_key
req_for_file = requests.get(image_url, stream=True)
# Init File-like object (to be used by upload_fileobj method)
file_object_from_req = req_for_file.raw
def send_topic(text):
producer = KafkaProducer(
bootstrap_servers='localhost:9092',
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
producer.send('nifi', key=b'text', value=text)
scrape_config_file = 'scrape.yml'
reddit_config = {
k: os.environ[k]
for k in
['client_id', 'client_secret', 'password', 'user_agent', 'username']
}
kafka_config = {k: os.environ[k] for k in ['host', 'port', 'topic']}
subreddit_names = os.environ['subreddits'].split(',')
print(f'kafka_config: {kafka_config}')
print(f'reddit_config: {reddit_config}')
print(f'subreddit_names: {subreddit_names}')
time.sleep(5)
producer = KafkaProducer(
api_version=(2, 6),
bootstrap_servers=f'{kafka_config["host"]}:{kafka_config["port"]}',
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
topic = kafka_config["topic"]
reddit = praw.Reddit(**reddit_config)
thread_list = []
for subreddit_name in subreddit_names:
thread = threading.Thread(target=print_submissions,
group=None,
args=(reddit, subreddit_name,
kafka_submission),
daemon=True)
thread_list.append(thread)
thread.start()
for thread in thread_list:
thread.join()
# in the file data/stream_apple.json
import tweepy
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import time
import argparse
import string
import config
import json
from kafka import KafkaProducer
import time
producer = KafkaProducer(bootstrap_servers='localhost:9092')
producer = KafkaProducer(
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
def get_parser():
"""Get parser for command line arguments."""
parser = argparse.ArgumentParser(description="Twitter Downloader")
parser.add_argument("-q",
"--query",
dest="query",
help="Query/Filter",
default='-')
parser.add_argument("-d",
"--data-dir",
dest="data_dir",
def __init__(self, host: str, port: int) -> None:
h = f'{host}:{port}'
self.kf_producer = KafkaProducer(bootstrap_servers=h)
import csv
from configparser import ConfigParser
config = ConfigParser()
config.read('config.ini')
TOPIC = config.get('kafka','topic')
BOOTSTRAP_SERVERS = config.get('kafka','bootstrap_servers')
FILEPATH = config.get('kafka','csv_filepath')
#creating producer object for ingest data to kafka topic
producer=KafkaProducer(bootstrap_servers=BOOTSTRAP_SERVERS,api_version=(0,10,1))
if __name__ == "__main__":
#ingesting data as json into kafka topic
with open(FILEPATH,'r') as csv_file:
csv_reader=csv.reader(csv_file,delimiter=',')
next(csv_reader)
msg={}
for row in csv_reader:
msg['Invoice']=row[0]
msg['StockCode']=row[1]
msg['Description']=row[2]
msg['Quantity']=row[3]
msg['InvoiceDate']=row[4]
msg['Price']=row[5]
msg['CustomerID']=row[6]
def __init__(self, bootstrap_servers, topic):
self.bootstrap_servers = bootstrap_servers
self.topic = topic
self.producer = KafkaProducer(bootstrap_servers=bootstrap_servers)
def _setup_kafka_producer(self, bootstrap_servers):
return KafkaProducer(bootstrap_servers=bootstrap_servers)
from time import sleep
from json import dumps
from kafka import KafkaProducer
producer = KafkaProducer(bootstrap_servers=['localhost:9092'],
value_serializer=lambda m: dumps(m).encode('ascii'))
for e in range(1000):
data = {'number': e}
print(data)
producer.send('test', value=data)
sleep(5)
def create_kafkaProducer():
return KafkaProducer(bootstrap_servers=kafkaServerEndPoint,
sasl_mechanism=kafkaAuth['sasl_mechanism'],
sasl_plain_username=kafkaAuth['sasl_plain_username'],
sasl_plain_password=kafkaAuth['sasl_plain_password'])
def create_producer(broker_ids):
producer = KafkaProducer(bootstrap_servers=broker_ids)
return producer
parser.add_argument('id', type=str)
parser.add_argument('user_id', type=str)
parser.add_argument('timestamp', type=lambda x: dateutil.parser.parse(x))
parser.add_argument('creation_timestamp',
type=lambda x: dateutil.parser.parse(x))
parser.add_argument('modified_timestamp',
type=lambda x: dateutil.parser.parse(x))
parser.add_argument('type', type=str)
parser.add_argument('data', type=dict)
basePath = Path(__file__).parent.parent / 'kafka_auth'
producer = KafkaProducer(
bootstrap_servers=['kafka-demo-parametrix-b70f.aivencloud.com:12744'],
value_serializer=lambda x: json.dumps(x).encode('utf-8'),
security_protocol="SSL",
ssl_cafile=basePath / "ca.pem",
ssl_certfile=basePath / "service.cert",
ssl_keyfile=basePath / "service.key",
api_version=(2, 5),
)
class EventApi(Resource):
@marshal_with(event_fields)
def get(self, event_id):
return Event.objects().get(id=event_id)
@marshal_with(event_fields)
def delete(self, event_id):
return Event.objects().get(id=event_id)
# Attempt connection
try:
ucscauthlogindb.connect()
print("Connected!")
except Exception as ex:
raise Exception(ex)
# Connect KafkaProducer
kafkaserver = [
"itsec-prod-elk-3.ucsc.edu:9092", "itsec-prod-elk-8.ucsc.edu:9092",
"itsec-prod-elk-9.ucsc.edu:9092"
]
topic = 'secinc'
try:
kproducer = KafkaProducer(bootstrap_servers=kafkaserver)
except Exception as ex:
raise Exception(ex)
# Set up time for data pull
nowDate = datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S")
pastDate = (datetime.datetime.today() -
datetime.timedelta(hours=24)).strftime("%Y-%m-%d %H:%M:%S")
authenticationsources = ['shibboleth', 'google', 'vpn']
# Loop through each authentication source and pull records related to potential compromises
for authsource in authenticationsources:
# Run query for the authentication source
results = ucscauthlogindb.getUserLoginsForAuthsource(
def __init__(self):
self.producer = KafkaProducer(bootstrap_servers='192.168.100.122',
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
#!/usr/bin/python3
# yum -y install python36-setuptools
# easy_install-3.6 pip
# pip install kafka kafka-python
from kafka import KafkaProducer
from kafka import KafkaConsumer
#Producer
producer = KafkaProducer(bootstrap_servers='127.0.0.1:9092')
future = producer.send('TopicName', b'hello_world')
result = future.get(timeout= 10)
#print(result)
#Consumer
consumer = KafkaConsumer('TopicName', group_id='GroupID', bootstrap_servers=['127.0.0.1:9092'])
for msg in consumer:
print(msg)
def __init__(self):
self._producer = KafkaProducer(
bootstrap_servers=config.CONFIG['kafka_broker'] + ":" +
config.CONFIG['port'])
from kafka import KafkaProducer
from time import sleep
import parsApi
import json
bootstarpServer = ['localhost:9092']
producer = KafkaProducer(
bootstrap_servers=bootstarpServer,
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
weather_data = parsApi.get_Api(city='london')
data_weather_forcast_5day = parsApi.get_date_weatherstatus_tmp_windspeed(
weather_data)
for e in range(len(data_weather_forcast_5day)):
temporary_data = data_weather_forcast_5day[e]
print('done')
producer.send('weather_forcast', temporary_data)
sleep(3)
producer.close()
from inferencer.YAMnet import YAMnet
from nivelDeRuido.nivelRuido import NivelRuido
from resources.recorder import recorder
from resources.deviceInfo import deviceInfo
logging.getLogger().setLevel(logging.INFO)
device = deviceInfo()
info = device.getInfoObj()
inferencer = YAMnet()
nivel_ruido = NivelRuido()
recorder = recorder(dirname='./', time=10)
inferencer_identifier = uuid.uuid4().__str__()
producer = KafkaProducer(
bootstrap_servers=[os.environ["KAFKA_BOOTSTRAP_SERVER_ONE"]],
value_serializer=lambda x: dumps(x).encode("utf-8"))
try:
while True:
filename = "raspberry-" + uuid.uuid1().__str__()
ruta = recorder.record(id=filename)
now = datetime.now(tz=tz.tzutc())
date_time = now.strftime("%Y-%m-%dT%H:%M:%S")
info['audio_uuid'] = filename
info['time'] = date_time
start = datetime.now()
inferencer_result = inferencer.run_inferencer(filename)
Leq = nivel_ruido.calcular_db(filename)
