def main():
context = SimpleStreamingContext()
ingestSerializer = KafkaStringSerializer()
emitSerializer = KafkaStringSerializer()
ingest = KafkaIngest(serializer=ingestSerializer,
kafka_topic=context.getVariable("import_key"),
bootstrap_server=context.getVariable("bootstrap_server"),
bootstrap_port=context.getVariable("bootstrap_port"))
process = BaseProcess()
emit = KafkaEmit(serializer=emitSerializer,
kafka_topic=context.getVariable("export_key"),
bootstrap_server=context.getVariable("bootstrap_server"),
bootstrap_port=context.getVariable("bootstrap_port"))
context.addIngest(ingest=ingest)
context.addEmit(emit=emit)
context.addProcess(process=process)
context.execute()
def main():
list_of_inserts = ['{"first_name":"jessica", "last_name":"st. german", "additional":"unneeded1"}',
'{"first_name":"jessica", "last_name":"courtney", "additional":"unneeded2"}',
'{"first_name":"jessica", "last_name":"mishra", "additional":"unneeded3"}',
'{"first_name":"jessica", "last_name":"novinha", "additional":"unneeded4"}',
'{"first_name":"jessica", "last_name":"tudor", "additional":"unneeded5"}',
'{"first_name":"jessica", "last_name":"ael-rayya", "additional":"unneeded6"}',
'{"first_name":"jessica", "last_name":"zuma", "additional":"unneeded7"}',
'{"first_name":"jessica", "last_name":"akihita", "additional":"unneeded8"}',
'{"first_name":"jessica", "last_name":"xi", "additional":"unneeded9"}',
'{"first_name":"jessica", "last_name":"kurylova", "additional":"unneeded10"}']
context = SimpleStreamingContext()
tuple_factory = SQLTupleFactory(tuple_fields=context.getVariable("sql_fields"))
ingest = SimulatedIngest(serializer=EmptyStringSerializer, stream_list=list_of_inserts)
process = ObjectToSQL(process=JsonToObject(), tuple_factory=tuple_factory)
emit = SQLEmit(serializer=EmptyObjectSerializer,
database=context.getVariable("database"),
host=context.getVariable("host"),
user=context.getVariable("user"),
password=context.getVariable("password"),
table=context.getVariable("table"),
tuple_factory=tuple_factory)
context.addIngest(ingest=ingest)
context.addProcess(process=process)
context.addEmit(emit=emit)
context.execute()
def main():
context = SimpleStreamingContext(
) # Choose an appropriate context, such as SimpleStreamingContext
ingest_serializer = KafkaStringSerializer() # e.g. KafkaStringSerializer
emit_serializer = StringSerializer() # e.g. StringSerializer
ingest = KafkaIngest(ingest_serializer,
kafka_topic=context.getVariable("import_key"),
bootstrap_server=context.getVariable(
"bootstrap_server")) # e.g. KafkaIngest
process = BaseProcess() # e.g. BaseProcess
emit = KafkaEmit(emit_serializer,
kafka_topic=context.getVariable("export_key"),
bootstrap_server=context.getVariable(
"bootstrap_server")) # e.g. KafkaEmit
context.addIngest(ingest=ingest) # Registers the ingest primitive
context.addProcess(process=process) # Registers the process primitive
context.addEmit(emit=emit) # Registers the emit primitive
context.execute() # Executes the context
def main():
list_of_inserts = [
'{"first_name":"jessica", "last_name":"st. german", "additional":"unneeded1"}',
'{"first_name":"jessica", "last_name":"courtney", "additional":"unneeded2"}',
'{"first_name":"jessica", "last_name":"mishra", "additional":"unneeded3"}',
'{"first_name":"jessica", "last_name":"novinha", "additional":"unneeded4"}',
'{"first_name":"jessica", "last_name":"tudor", "additional":"unneeded5"}',
'{"first_name":"jessica", "last_name":"ael-rayya", "additional":"unneeded6"}',
'{"first_name":"jessica", "last_name":"zuma", "additional":"unneeded7"}',
'{"first_name":"jessica", "last_name":"akihita", "additional":"unneeded8"}',
'{"first_name":"jessica", "last_name":"xi", "additional":"unneeded9"}',
'{"first_name":"jessica", "last_name":"kurylova", "additional":"unneeded10"}'
]
context = SimpleStreamingContext()
ingest = SimulatedIngest(serializer=EmptyStringSerializer,
stream_list=list_of_inserts)
emit_serializer = KafkaStringSerializer() # e.g. StringSerializer
process = BaseProcess() # e.g. BaseProcess
emit = KafkaEmit(emit_serializer,
kafka_topic=context.getVariable("export_key"),
bootstrap_server=context.getVariable(
"bootstrap_server")) # e.g. KafkaEmit
context.addIngest(ingest=ingest) # Registers the ingest primitive
context.addProcess(process=process) # Registers the process primitive
context.addEmit(emit=emit) # Registers the emit primitive
context.execute() # Executes the context
def main():
list_of_inserts = ['{"actor_id":210, "first_name":"jess", "last_name":"st. german", "additional":"unneeded1"}',
'{"actor_id":201, "first_name":"jess", "last_name":"courtney", "additional":"unneeded2"}',
'{"actor_id":202, "first_name":"jess", "last_name":"mishra", "additional":"unneeded3"}',
'{"actor_id":203, "first_name":"jess", "last_name":"novinha", "additional":"unneeded4"}',
'{"actor_id":204, "first_name":"jess", "last_name":"changed", "additional":"unneeded5"}',
'{"actor_id":205, "first_name":"jess", "last_name":"ael-rayya", "additional":"unneeded6"}',
'{"actor_id":206, "first_name":"jess", "last_name":"zuma", "additional":"unneeded7"}',
'{"actor_id":207, "first_name":"jess", "last_name":"changed", "additional":"unneeded8"}',
'{"actor_id":208, "first_name":"jess", "last_name":"changed", "additional":"unneeded9"}',
'{"actor_id":209, "first_name":"jess", "last_name":"changed", "additional":"unneeded10"}']
context = SimpleStreamingContext() # Choose an appropriate context, such as SimpleStreamingContext
ingest_serializer = EmptyStringSerializer() # e.g. KafkaStringSerializer
emit_serializer = KafkaStringSerializer() # e.g. StringSerializer
ingest = SimulatedIngest(serializer=ingest_serializer, stream_list=list_of_inserts)
p1 = JsonToObject()
p1 = KeyedFilter(key="actor_id", filter_callable=filteractorid, process=p1)
p1 = ObjectToJson(process=p1)
emit = KafkaEmit(emit_serializer,
kafka_topic=context.getVariable("export_key"),
bootstrap_server=context.getVariable("bootstrap_server"),
emit_buffer_batch_size=1) # e.g. KafkaEmit
context.addIngest(ingest=ingest) # Registers the ingest primitive
context.addProcess(process=p1) # Registers the process primitive
context.addEmit(emit=emit) # Registers the emit primitive
context.execute() # Executes the context
def main():
context = SimpleStreamingContext()
ingest_serializer = EmptyStringSerializer
emit_serializer = EmptyStringSerializer
ingest = TwitterStreamingIngest(
serializer=ingest_serializer,
bearer_token=context.getVariable("bearer_token"),
tweet_fields=context.getVariable("tweet_fields"),
user_fields=context.getVariable("user_fields"),
place_fields=context.getVariable("place_fields"),
media_fields=context.getVariable("media_fields"))
process = BaseProcess()
emit = StdoutEmit(serializer=emit_serializer)
context.addIngest(ingest)
context.addProcess(process)
context.addEmit(emit)
context.execute()
def main():
context = SimpleStreamingContext()
ingest_serializer = EmptyStringSerializer
emit_serializer = EmptyStringSerializer
ingest = CTPStreamingIngest(
serializer=ingest_serializer,
location=context.getVariable("location"),
timeframe=context.getVariable("timeframe"),
request_fields=context.getVariable("request_fields"))
process = BaseProcess()
emit = StdoutEmit(serializer=emit_serializer)
context.addIngest(ingest)
context.addProcess(process)
context.addEmit(emit)
context.execute()
def main():
context = SimpleStreamingContext()
ingest_serializer = EmptyStringSerializer
emit_serializer = EmptyStringSerializer
ingest = CDCStreamingIngest(serializer=ingest_serializer,
cdc_fields=context.getVariable("cdc_fields"))
process = BaseProcess()
emit = StdoutEmit(serializer=emit_serializer)
context.addIngest(ingest)
context.addProcess(process)
context.addEmit(emit)
context.execute()