def __init__(self, brokers, n_workers=5, topic_work=mjolnir.kafka.TOPIC_REQUEST, topic_result=mjolnir.kafka.TOPIC_RESULT, topic_complete=mjolnir.kafka.TOPIC_COMPLETE, max_request_size=4 * 1024 * 1024): self.brokers = brokers self.n_workers = n_workers self.topic_work = topic_work self.topic_result = topic_result self.topic_complete = topic_complete # Standard producer for query results self.producer = kafka.KafkaProducer( bootstrap_servers=brokers, max_request_size=max_request_size, compression_type='gzip', api_version=mjolnir.kafka.BROKER_VERSION) # More reliable producer for reflecting end run sigils. As this # is only used for sigils and not large messages like es responses # compression is unnecessary here. self.ack_all_producer = kafka.KafkaProducer( bootstrap_servers=brokers, acks='all', api_version=mjolnir.kafka.BROKER_VERSION) # TODO: 10 items? No clue how many is appropriate...10 seems reasonable # enough. We want enough to keep the workers busy, but not so many # that the commited offsets are siginficantly ahead of the work # actually being performed. self.work_queue = Queue.Queue(10)
def __init__(self, *, config): self.log = logging.getLogger(self.__class__.__name__) self.config = config object_storage_config = self.config.get("object_storage", {}) object_storage_type = object_storage_config.get("type") if object_storage_type == "gcs": self.object_storage = GCSProvider(config=object_storage_config) else: raise ValueError( f"Unknown object storage type: {object_storage_type}") kafka_config = self.config.get("kafka", {}) if ("ssl_cafile" in kafka_config and "ssl_access_certificate_file" in kafka_config and "ssl_access_key_file" in kafka_config): self.kafka_producer = kafka.KafkaProducer( bootstrap_servers=kafka_config["kafka_url"], security_protocol="SSL", ssl_cafile=kafka_config["ssl_ca_file"], ssl_certfile=kafka_config["ssl_access_certificate_file"], ssl_keyfile=kafka_config["ssl_access_key_file"], ) else: self.kafka_producer = kafka.KafkaProducer( bootstrap_servers=kafka_config["kafka_url"], )
def publishKafkaData(self, jsonData): """ Publish data to Kafka topic """ logging.debug("Executing sendStatistics.publishKafkaData") logging.info("Sending data to Kafka topic") logging.debug("Sending the following JSON to Kafka: %s" % (jsonData)) kafkaError = False result = True # logging.error("SEND TO KAFKA DISABLED") # return result try: ssl_context = ssl.create_default_context( cafile=self.kafkaTrustCAFile) except FileNotFoundError: logging.error("SSL CA file for Kafka cant be found") ssl_context = None try: if self.kafkaSaslMechanism != None and self.kafkaSecurityProtocol != None: producer = kafka.KafkaProducer( bootstrap_servers=self.kafkaBrokers, security_protocol=self.kafkaSecurityProtocol, sasl_mechanism=self.kafkaSaslMechanism, ssl_context=ssl_context) else: producer = kafka.KafkaProducer( bootstrap_servers=self.kafkaBrokers, ssl_context=ssl_context) producer.send(topic=self.kafkaTopic, value=jsonData.encode('utf-8')) except AssertionError as e: logging.error("Kafka Error: %s" % e) kafkaError = True except kafka.errors.NoBrokersAvailable: logging.error("None of the Kafka brokers configured are available") kafkaError = True except kafka.errors.UnrecognizedBrokerVersion: logging.error( "Unrecognized broker version. Are you connecting to a secure Kafka broker without the correct security protocols?" ) kafkaError = True if kafkaError == True: result = False logging.debug("Executing sendStatistics.publishKafkaData - Finished") return result
def _create_producer(self) -> None: import kafka if self.security_protocol == "PLAINTEXT": self.producer = kafka.KafkaProducer( client_id=self.client_id, bootstrap_servers=self.url, value_serializer=lambda v: json.dumps(v).encode( DEFAULT_ENCODING), security_protocol=self.security_protocol, ssl_check_hostname=False, ) elif self.security_protocol == "SASL_PLAINTEXT": self.producer = kafka.KafkaProducer( client_id=self.client_id, bootstrap_servers=self.url, value_serializer=lambda v: json.dumps(v).encode( DEFAULT_ENCODING), sasl_plain_username=self.sasl_username, sasl_plain_password=self.sasl_password, sasl_mechanism="PLAIN", security_protocol=self.security_protocol, ) elif self.security_protocol == "SSL": self.producer = kafka.KafkaProducer( client_id=self.client_id, bootstrap_servers=self.url, value_serializer=lambda v: json.dumps(v).encode( DEFAULT_ENCODING), ssl_cafile=self.ssl_cafile, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile, ssl_check_hostname=False, security_protocol=self.security_protocol, ) elif self.security_protocol == "SASL_SSL": self.producer = kafka.KafkaProducer( client_id=self.client_id, bootstrap_servers=self.url, value_serializer=lambda v: json.dumps(v).encode( DEFAULT_ENCODING), sasl_plain_username=self.sasl_username, sasl_plain_password=self.sasl_password, ssl_cafile=self.ssl_cafile, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile, ssl_check_hostname=self.ssl_check_hostname, security_protocol=self.security_protocol, sasl_mechanism="PLAIN", ) else: raise ValueError( f"Cannot initialise `KafkaEventBroker`: " f"Invalid `security_protocol` ('{self.security_protocol}').")
def __init__(self, config: Dict[str, Any]) -> None: token = config['slack']['api_token'] self.slack_client = BlockingSlackClient(token) self.destination_kafka_topic = config['kafka']['from_slack']['topic'] self.kafka_producer = kafka.KafkaProducer( bootstrap_servers=config['kafka']['bootstrap_servers'], )
def __init__(self, client_id, host, port, ca_file, cert_file, key_file, kafka_topic): # Set up kafka producer self.producer = kafka.KafkaProducer( bootstrap_servers="{}:{}".format(host, port), client_id=client_id, security_protocol="SSL", ssl_cafile=ca_file, ssl_certfile=cert_file, ssl_keyfile=key_file, ) self.client_id = client_id self.kafka_topic = kafka_topic self.cpu_percentages = None self.memory_percentages = None # Get basic OS information self.mac_id = ':'.join(re.findall('..', '%012x' % uuid.getnode())) self.os_name = platform.system() self.os_release = platform.release() self.os_architecture = "{} ({})".format(platform.architecture()[0], platform.machine()) self.os_version = platform.version() # Get information about processor and memory self.processor = platform.processor() self.cpu_physical_cores = psutil.cpu_count(logical=False) self.cpu_logical_cores = psutil.cpu_count(logical=True) self.total_ram = psutil.virtual_memory().total self.total_swap_space = psutil.swap_memory().total
def a(): # 正常发送,没有主动调用close是不会发送的,是强制退出 producer = kafka.KafkaProducer(**conf) for i in range(3): producer.send('partopic', str(i).encode()) logging.info('手动触发close----------') producer.close()
def _main(camid, mjpg, kafka, topic): stream = urllib.urlopen(mjpg) bytes = '' while True: bytes += stream.read(1024) a = bytes.find('\xff\xd8') b = bytes.find('\xff\xd9') if a != -1 and b != -1: jpg = bytes[a:b + 2] bytes = bytes[b + 2:] frame = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_COLOR) payload = { 'camera_id': camid, 'timestamp': int(time.time()), 'rows': frame.shape[0], 'cols': frame.shape[1], 'type': 'uint8', 'data': base64.b64encode(jpg) } producer = kafka_client.KafkaProducer(bootstrap_servers=kafka, batch_size=512000, api_version=(0, 10, 1)) producer.send(topic, key=camid, value=json.dumps(payload))
def main(argv=sys.argv): """ Produce simulated data to the kafka for stress test Return: Void """ producer = kafka.KafkaProducer(bootstrap_servers=PUBLIC_DNS) periodic_request(producer, TIME_PERIOD)
def stopKafkaProducer(): import kafka producer_id = "diskeyteststop" producer = kafka.KafkaProducer(bootstrap_servers='192.168.1.130:9092') producer.flush() producer.close() return json.dumps({'status': 'OK stop producer','producer': producer_id})
def __init__(self, topic=settings.DEFAULT_TOPIC): log.debug("producer.init") self.topic = topic self.producer = kafka.KafkaProducer( value_serializer=lambda m: json.dumps(m).encode("utf-8"), bootstrap_servers=settings.KAFKA_SERVER) log.debug("producer.created")
def producer(args): # connect to kafka producer = kafka.KafkaProducer( bootstrap_servers=args.kafka_brokers.split(","), partitioner=partitioner) # initialize packet capture capture = pcapy.open_live(args.interface, 65535, True, 3000) packet_count = 0 # start packet capture while True: (pkt_hdr, pkt_raw) = capture.next() if pkt_hdr is not None: # send packet to kafka pkt_ts = timestamp(pkt_hdr) producer.send(args.topic, key=pack_ts(pkt_ts), value=pkt_raw) # debug messages, if needed packet_count += 1 if args.debug > 0 and packet_count % args.debug == 0: print 'Sent Packet: count=%s dt=%s topic=%s' % ( packet_count, to_date(pkt_ts), args.topic) print to_hex(pkt_raw)
def __init__(self, hosts='127.0.0.1:9092', topic='default', timeout=None, compression=None, acks=1, retries=0, job_size=1048576, cafile=None, certfile=None, keyfile=None, crlfile=None): self._hosts = hosts self._topic = topic self._timeout = timeout self._logger = logging.getLogger('kq') self._producer = kafka.KafkaProducer( bootstrap_servers=self._hosts, compression_type=compression, acks=acks, retries=retries, max_request_size=job_size, buffer_memory=max(job_size, 33554432), ssl_cafile=cafile, ssl_certfile=certfile, ssl_keyfile=keyfile, ssl_crlfile=crlfile )
def test_create_producer(self): print("test_create_producer") is_exception = False config_file = "config.ini" config = configparser.ConfigParser() config.read(config_file) try: kafka_config = config["kafka"] topic = kafka_config["topic"] sleep_second = int(kafka_config["sleep"]) producer_config = config["producer"] records = int(producer_config["records"]) except KeyError as e: print("Config file missing section or key \"" + e.args[0] + "\"") is_exception = True try: producer = kafka.KafkaProducer( bootstrap_servers=kafka_config["host"] + ":" + kafka_config["port"], security_protocol="SSL", ssl_cafile=kafka_config["cafile"], ssl_certfile=kafka_config["certfile"], ssl_keyfile=kafka_config["keyfile"], ) except KeyError as e: print("Kafka config missing key \"" + e.args[0] + "\"") is_exception = True self.assertEqual(False, is_exception)
def task_2(): try: connection = psycopg2.connect( user="******", password="******", host="localhost", port="5432", database="test_aut", ) cursor = connection.cursor() postgresSQL_select_Query = "select * from table_1" cursor.execute(postgresSQL_select_Query) user_record = cursor.fetchall() # print(user_record) producer = kafka.KafkaProducer( bootstrap_servers=["localhost:9092"], value_serializer=lambda x: dumps(x).encode("utf-8"), ) for record in user_record: id_, name1, name2, name3, name4, name5 = record print(name1) data = {"name": name1} producer.send("trump", value=data) except (Exception, psycopg2.Error) as error: print("Error:::::", str(error)) else: if connection: cursor.close() connection.close() print("connection closed")
def face_submit(): ans = {'status': 200, 'err_msg': ''} try: face_name1 = flask.request.form.get('face_name1') face_name2 = flask.request.form.get('face_name2') print(face_name1, face_name2) info = { 'id': str(uuid.uuid1()), 'face_name1': face_name1, 'face_name2': face_name2, 'status': 'unfinished', 'owner': flask_login.current_user.id, 'create_date': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) } info_str = json.dumps(info) producer = kafka.KafkaProducer( bootstrap_servers=[configs.app_kafka_host]) future = producer.send(configs.app_kafka_topic, key=bytes(configs.app_kafka_key, encoding='utf-8'), value=bytes(info_str, encoding='utf-8')) producer.close() try: future.get(timeout=5) # 监控是否发送成功 except Exception as e: # 发送失败抛出kafka_errors traceback.print_exc() ans['id'] = info['id'] ans['face_name1'] = info['face_name1'] ans['face_name2'] = info['face_name2'] except Exception as e: ans['status'] = 500 ans['err_msg'] = str(e) return ans
def connect(self, *args: Any, **kwargs: Any) -> kafka.KafkaProducer: if 'value_serializer' not in self.config and 'value_serializer' not in kwargs: kwargs['value_serializer'] = lambda x: json.dumps(x).encode('utf-8' ) self.session = kafka.KafkaProducer(**self.config, **kwargs) self._closed = False return self.session
def createProducer(**configOptions): """This initializes a Kafka producer for use in the main function""" while(True): # Checking to see if the user wants to use a non-default address while(True): brokerId = input('Would you like to connect to a non-default broker address? su[Default: localhost:9092] (Y/N)') if(brokerId.lower().startswith('y')): brokerId = input("Please enter your broker address in the format 'host:port' ") break elif(brokerId.lower().startswith('n')): brokerId = 'localhost:9092' break # Configuring the producer with the necessary arguments # The value serializer is given a lambda that converts the sent data into bytes for submission to the cluster producer = kf.KafkaProducer(client_id="pii-generator", bootstrap_servers=[brokerId], value_serializer=lambda v: json.dumps(v).encode('utf-8')) # ! Make sure that this break statement is not cutting the creation of a producer off if(producer.bootstrap_connected()): print("Successfully connected to bootstrap server...") break return producer
def __init__(self,config_file): super().__init__() self.args = AlphaConfig.loadConfig( os.path.join( AlphaPathLookUp.ConfigPath, config_file ) ) #self.is_ready = Event() self.producer=kafka.KafkaProducer(bootstrap_servers=self.args.producer["servers"],value_serializer= lambda m: json.dumps(m).encode('ascii') ) self.consumer=kafka.KafkaConsumer(self.args.consumer["topic"], group_id=self.args.consumer["group_id"], bootstrap_servers=self.args.consumer["servers"], value_deserializer= lambda m: json.loads(m.decode("ascii")))
def doPreinit(self, mode): if mode != SIMULATION: self._producer = kafka.KafkaProducer( bootstrap_servers=self.brokers, max_request_size=self.max_request_size) else: self._producer = None
def main(consumer_topic=TOPIC, producer_topic=VALIDATION_TOPIC, boot_servers=BOOT_SERVERS): consumer = kafka.KafkaConsumer(consumer_topic, bootstrap_servers=boot_servers, auto_offset_reset="earliest", value_deserializer=_deserializer) producer = kafka.KafkaProducer( bootstrap_servers=boot_servers, value_serializer=lambda m: json.dumps(m).encode('utf-8')) def validate_data(request_id, result="success"): msg = {"request_id": request_id, "validation": result} producer.send(producer_topic, msg) for msg in consumer: data = msg.value if not data or not isinstance(data, dict): # smth is wrong, not processing continue reports = fetch_report(data["url"]) if not reports: # couldn't fetch report data, discarding message validate_data(data.get("request_id"), result="handoff") else: LOG.info("Fetched and validated tgz from %s", data["url"]) # data is valid, waving green flag validate_data(data.get("request_id")) # process data (upload to db in our case) for report in reports: res = store_in_db(report) LOG.info("Uploaded: %s", json.dumps(res).encode('utf-8'))
def entry_point(): params = parse_args() if params.cp == ConsumerProducer.PRODUCER: producer = kafka.KafkaProducer( bootstrap_servers=[params.addr], value_serializer=lambda x: json.dumps(x).encode('utf-8')) while True: for i in range(0, sys.maxsize): data = {'number': i} producer.send(params.topic, value=data) time.sleep(1) else: consumer = kafka.KafkaConsumer( params.topic, bootstrap_servers=[params.addr], group_id='my-group', value_deserializer=lambda x: json.loads(x.decode('utf-8'))) for message in consumer: message = message.value print("message is %s" % message) pass
def run(self): i = 0 try: producer = kafka.KafkaProducer( bootstrap_servers= 'broker-0.broker.default.svc.cluster.local:9092') except kafka.errors.NoBrokersAvailable: time.sleep(30) self.run() while not self.stop_event.is_set(): # Send a job in the kafka queue offset = i + 1 requests.get( "http://pacman:80/metrics/" + str(offset) ) # Request pacman to tell him that a new job is in the queue producer.send('topic_1', ("ciao" + str(i)).encode()) i += 1 #w = random() #t = np.exp(w) b = 1.0 t = np.random.exponential(b) time.sleep(t) producer.close()
def __init__(self, connection_info, advanced_info, topic_in, topic_out, predictor, _type): self.connection_info = connection_info self.advanced_info = advanced_info self.predictor = predictor self.stream_in_name = topic_in self.stream_out_name = topic_out self.consumer = kafka.KafkaConsumer( **self.connection_info, **self.advanced_info.get('consumer', {})) self.consumer.subscribe(topics=[self.stream_in_name]) self.producer = kafka.KafkaProducer( **self.connection_info, **self.advanced_info.get('producer', {})) self.admin = kafka.KafkaAdminClient(**self.connection_info) try: self.topic = NewTopic(self.stream_out_name, num_partitions=1, replication_factor=1) self.admin.create_topics([self.topic]) except kafka.errors.TopicAlreadyExistsError: pass self._type = _type self.native_interface = NativeInterface() self.format_flag = 'explain' self.stop_event = Event() self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None) self.caches = {} if self._type == 'timeseries': super().__init__(target=KafkaStream.make_timeseries_predictions, args=(self, )) else: super().__init__(target=KafkaStream.make_prediction, args=(self, ))
def sendmessage(host, label, series, message): if kafka.__version__.startswith("0"): c = kafka.client.KafkaClient(hosts=[host]) if series: p = kafka.producer.keyed.KeyedProducer(c) else: p = kafka.producer.simple.SimpleProducer(c) else: p = kafka.KafkaProducer(bootstrap_servers=host) success = False t = 0.2 while not success: try: if kafka.__version__.startswith("0"): if series: p.send_messages(label, series.encode("utf-8"), message.encode("utf-8")) else: p.send_messages(label, message.encode("utf-8")) else: p.send(label, key=series.encode("utf-8"), value=message.encode("utf-8")) p.close() print("success") success = True except Exception as e: print("error (sleep {})".format(t), e) time.sleep(t) t *= 2
def __init__(self, *args): super(Srv, self).__init__(*args) self.ListenForEvent(serverApi.GetEngineNamespace(), serverApi.GetEngineSystemName(), 'AddServerPlayerEvent', self, self.connection_made) self.ListenForEvent(serverApi.GetEngineNamespace(), serverApi.GetEngineSystemName(), 'DelServerPlayerEvent', self, self.connection_lost) self.ListenForEvent(serverApi.GetEngineNamespace(), serverApi.GetEngineSystemName(), 'LoadServerAddonScriptsAfter', self, self.serve_forever) self.ListenForEvent(cfg.MOD_NAMESPACE, cfg.MOD_CLI_NAME, 'G_DEBUT', self, self.debut) self.ListenForEvent(cfg.MOD_NAMESPACE, cfg.MOD_CLI_NAME, 'G_MATCH', self, self.match) self.ListenForEvent(cfg.MOD_NAMESPACE, cfg.MOD_CLI_NAME, 'G_COURT', self, self.rcall) self.CreateComponent(serverApi.GetLevelId(), 'Minecraft', 'game').OpenCityProtect() self.s = set() self._mapping = {} self._q = Queue() self._alive = True self._consumer = kafka.KafkaConsumer('duel', bootstrap_servers=KAFKA_SERVERS, value_deserializer=pickle.loads) self._producer = kafka.KafkaProducer(bootstrap_servers=KAFKA_SERVERS, value_serializer=pickle.dumps) redisPool.InitDB(10) mysqlPool.InitDB(20)
def _send_message(self, mes, topic): ''' Send the message to next topic :param mes{map}: the message need to be sent :param topic{string}: the topic of the message ''' self.logger.info(mes) self.logger.info(type(mes)) self.logger.info(topic) try: mesg = str(json.dumps(mes)).encode('utf-8') # mesg = json.dumps(mes) producer = kafka.KafkaProducer( bootstrap_servers=self.kafka_cluster) # if producer: producer.send(topic, mesg) self.logger.info("Send the message to next topic successully!") producer.close() except Exception: self.logger.error( "Errors occured while sending message to next topic") if producer: producer.close() raise
def Produce(self, topic, value): """ 生产消息 topic: 主题 value: 值, 二进制值, 如:b'12' """ try: if isinstance(value, str): print('Produce invalid data: %s:%s' % (topic, value)) value = value.encode('utf8') elif isinstance(value, bytes) is not True: print('Produce invalid data: %s:%s' % (topic, value)) return err = None for i in range(3): try: self.producer.send(topic, value) return True except Exception as e: err = e self.producer = kafka.KafkaProducer( bootstrap_servers=self.brokers) Log(LOG_ERROR, "KafkaProducer", '向kafka(%s)发送消息%s异常: %s' % (topic, value, err)) return False except Exception as err: Log(LOG_ERROR, "KafkaProducer", '向kafka(%s)发送消息%s异常: %s' % (topic, value, err)) return False
def start(self): try: self.producer = kafka.KafkaProducer( bootstrap_servers=self.bootstrap_servers) except kafka.errors.NoBrokersAvailable as err: log.error(err, exc_info=True) raise NapalmLogsException(err)
def _main(base_word, kafka, topic): """TweePub reads tweets from Twitter Streaming API with provided characteristics and pushes them to specified Apache Kafka instance. """ click.echo("Using base word: %s" % base_word) click.echo("Kafka bootstrap servers: %s" % kafka) producer = kafka_client.KafkaProducer(bootstrap_servers=kafka, value_serializer=str.encode) for _ in range(0, 100): hashtags = [] for i in range(1, 16): if random.randint(1, i+1) == i: hashtags.append(base_word + "-" + str(i)) click.echo("Pushing hashtags: %s" % hashtags) producer.send(topic, json.dumps(dict( entities=dict( hashtags=[ dict(text=ht) for ht in hashtags ] ) ))) time.sleep(5) producer.flush()