def main(): list_of_inserts = [ '{"first_name":"jessica", "last_name":"st. german", "additional":"unneeded1"}', '{"first_name":"jessica", "last_name":"courtney", "additional":"unneeded2"}', '{"first_name":"jessica", "last_name":"mishra", "additional":"unneeded3"}', '{"first_name":"jessica", "last_name":"novinha", "additional":"unneeded4"}', '{"first_name":"jessica", "last_name":"tudor", "additional":"unneeded5"}', '{"first_name":"jessica", "last_name":"ael-rayya", "additional":"unneeded6"}', '{"first_name":"jessica", "last_name":"zuma", "additional":"unneeded7"}', '{"first_name":"jessica", "last_name":"akihita", "additional":"unneeded8"}', '{"first_name":"jessica", "last_name":"xi", "additional":"unneeded9"}', '{"first_name":"jessica", "last_name":"kurylova", "additional":"unneeded10"}' ] context = SimpleStreamingContext() ingest = SimulatedIngest(serializer=EmptyStringSerializer, stream_list=list_of_inserts) emit_serializer = KafkaStringSerializer() # e.g. StringSerializer process = BaseProcess() # e.g. BaseProcess emit = KafkaEmit(emit_serializer, kafka_topic=context.getVariable("export_key"), bootstrap_server=context.getVariable( "bootstrap_server")) # e.g. KafkaEmit context.addIngest(ingest=ingest) # Registers the ingest primitive context.addProcess(process=process) # Registers the process primitive context.addEmit(emit=emit) # Registers the emit primitive context.execute() # Executes the context
def main(): context = SimpleStreamingContext() ingest_serializer = EmptyStringSerializer emit_serializer = EmptyStringSerializer ingest = CDCStreamingIngest(serializer=ingest_serializer, cdc_fields=context.getVariable("cdc_fields")) process = BaseProcess() emit = StdoutEmit(serializer=emit_serializer) context.addIngest(ingest) context.addProcess(process) context.addEmit(emit) context.execute()
def main(): context = EdnaContext( ) # Choose an appropriate context, such as SimpleStreamingContext ingest_serializer = Serializable() # e.g. KafkaStringSerializer emit_serializer = Serializable() # e.g. StringSerializer ingest = BaseIngest(ingest_serializer, context.getVariable("VARIABLE1")) # e.g. KafkaIngest process = BaseProcess(context.getVariable("VARIABLE2")) # e.g. BaseProcess emit = BaseEmit(emit_serializer, context.getVariable("VARIABLE1")) # e.g. KafkaEmit context.addIngest(ingest=ingest) # Registers the ingest primitive context.addProcess(process=process) # Registers the process primitive context.addEmit(emit=emit) # Registers the emit primitive context.execute() # Executes the context
def main(): context = SimpleStreamingContext() # Choose an appropriate context, such as SimpleStreamingContext ingest_serializer = StringSerializer() # e.g. KafkaStringSerializer emit_serializer = KafkaStringSerializer() # e.g. StringSerializer ingest = KafkaIngest(ingest_serializer, kafka_topic=context.getVariable("import_key"), bootstrap_server=context.getVariable("bootstrap_server")) # e.g. KafkaIngest process = BaseProcess() # e.g. BaseProcess emit = KafkaEmit(emit_serializer, kafka_topic=context.getVariable("export_key"), bootstrap_server=context.getVariable("bootstrap_server")) # e.g. KafkaEmit context.addIngest(ingest=ingest) # Registers the ingest primitive context.addProcess(process=process) # Registers the process primitive context.addEmit(emit=emit) # Registers the emit primitive context.execute() # Executes the context
def main(): context = SimpleStreamingContext() ingest_serializer = EmptyStringSerializer emit_serializer = EmptyStringSerializer ingest = CTPStreamingIngest( serializer=ingest_serializer, location=context.getVariable("location"), timeframe=context.getVariable("timeframe"), request_fields=context.getVariable("request_fields")) process = BaseProcess() emit = StdoutEmit(serializer=emit_serializer) context.addIngest(ingest) context.addProcess(process) context.addEmit(emit) context.execute()
def main(): context = SimpleStreamingContext() ingestSerializer = KafkaStringSerializer() emitSerializer = KafkaStringSerializer() ingest = KafkaIngest(serializer=ingestSerializer, kafka_topic=context.getVariable("import_key"), bootstrap_server=context.getVariable("bootstrap_server"), bootstrap_port=context.getVariable("bootstrap_port")) process = BaseProcess() emit = KafkaEmit(serializer=emitSerializer, kafka_topic=context.getVariable("export_key"), bootstrap_server=context.getVariable("bootstrap_server"), bootstrap_port=context.getVariable("bootstrap_port")) context.addIngest(ingest=ingest) context.addEmit(emit=emit) context.addProcess(process=process) context.execute()
def main(): context = SimpleStreamingContext() ingest_serializer = EmptyStringSerializer emit_serializer = EmptyStringSerializer ingest = TwitterStreamingIngest( serializer=ingest_serializer, bearer_token=context.getVariable("bearer_token"), tweet_fields=context.getVariable("tweet_fields"), user_fields=context.getVariable("user_fields"), place_fields=context.getVariable("place_fields"), media_fields=context.getVariable("media_fields")) process = BaseProcess() emit = StdoutEmit(serializer=emit_serializer) context.addIngest(ingest) context.addProcess(process) context.addEmit(emit) context.execute()