def get_from_pipeline(componentMap, **arg):
pipeline_name = arg['pipelineName']
component_name = arg['componentName']
pipeline = pkg.components.processComponentMap[pipeline_name]
message = pipeline.get_message()
log.info(f"Get {message} from pipeline. [size: {pipeline.qsize()}]")
componentMap[component_name] = message
def initialize():
pkg.components.processComponentMap["rules"] = read_yaml('../../rules/' + pkg.components.processComponentMap["processName"] + '.yaml')
initializers=pkg.components.processComponentMap["rules"]["Initializers"]
for initializer in initializers.values():
extension_map[initializer["extension"]].process(function=initializer["operation"], componentMap=pkg.components.processComponentMap,**initializer)
status=pkg.components.processComponentMap["status"]
log.info(status)
def consumer(pipeline_name, consumer_name, event_name, worker_id):
"""Pretend we're saving a number in the database."""
processRules = pkg.components.processComponentMap["rules"][consumer_name]
pipeline = pkg.components.processComponentMap[pipeline_name]
event = pkg.components.processComponentMap[event_name]
componentMap = {}
while not event.is_set() or not pipeline.empty():
log.info(f"Starting consumer {consumer_name}-{worker_id}")
for rule in processRules["extensionList"].values():
extension_map[rule["extension"]].process(
function=rule["operation"], componentMap=componentMap, **rule)
log.info(f"{consumer_name}-{worker_id} received EXIT event. Exiting")
def send(componentMap,**arg):
processComponentName = arg["processComponentName"]
now = datetime.datetime.now()
nowDict={}
nowDict["time"] = now.strftime("%c")
msg = json.dumps(componentMap[arg["componentName"]])
msgtime=json.dumps(nowDict)
log.info(f"Msg to send: {msg}")
kafkaProducer=pkg.components.processComponentMap[processComponentName]
future = kafkaProducer.send(arg["topicName"], msg.encode('ascii'))
future = kafkaProducer.send(arg["topicName"], msgtime.encode('ascii'))
kafkaProducer.flush()
def fileSystemEventHandler(event):
log.info(f"Processing file {event.src_path}")
# if event.eventType == 'created' or event.eventType == 'modified':
pName = pkg.components.processComponentMap["processName"]
processRules=pkg.components.processComponentMap["rules"][pName]
time.sleep(2)
with open(event.src_path, 'r') as file:
csvRec = csv.DictReader(file)
for rec in csvRec:
componentMap={}
componentMap[processRules["componentName"]] = rec
for rule in processRules["extensionList"].values():
extname = rule["extension"]
extension_map[rule["extension"]].process(function=rule["operation"],componentMap=componentMap, **rule)
os.remove(event.src_path)
def fileWatcher():
observer = Observer()
eventHandler = FileSystemEventHandler()
eventHandler.on_created = fileSystemEventHandler
#eventHandler.on_modified = fileSystemEventHandler
pName=pkg.components.processComponentMap["processName"]
observer.schedule(eventHandler, pkg.components.processComponentMap["rules"][pName]["directory"], recursive=True)
observer.start()
log.info("File watcher started")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def add_to_pipeline(componentMap, **arg):
pipeline_name = arg['pipelineName']
pipeline_size = arg['pipelineSize']
list_component_name = arg['listComponentName']
list_component = componentMap[list_component_name] if list_component_name in componentMap else []
pipeline = pkg.components.processComponentMap[pipeline_name]
length = len(list_component)
i = 0
while i < length:
if pipeline.qsize() < pipeline_size:
pipeline.set_message(list_component[i])
log.info(f"Added {list_component[i]} to pipeline [i={i}, size: {pipeline.qsize()}]")
i = i+1
else:
log.info(f"Pipeline is full, sleeping for 1s [i={i}, size: {pipeline.qsize()}]")
time.sleep(1)
def print(componentMap, **arg):
if "componentName" in arg:
if arg["componentName"] in componentMap:
log.info(componentMap[arg["componentName"]])
else:
log.info("Component " + arg["componentName"] +
" does not exist in the map")
else:
log.info(componentMap)
def produce_employee_profile(componentMap, **arg):
log.info("get the employee id list here, api fetch started")
time.sleep(5) #simulate an api fetch call delay
log.info("api fetch complete.")
arr = []
for x in range(100,
120): #simulating a db/api fetch to get list of emp ids
arr.append(x)
log.info(f"Adding {str(arr)} to component map from producer")
componentMap[arg['listComponent']] = arr
def initialize(componentMap, **arg):
log.info("Initialize something here")
def consume_company_details(componentMap, **arg):
element = componentMap['employeePipelineElement']
log.info(f"get the company details here for the employee id {element}")
# Do some other api fetch with element to receive additional data
time.sleep(3)
def set_event(componentMap, **arg):
pipeline_name = arg['pipelineName']
event_name = pipeline_name + '-event'
event = pkg.components.processComponentMap[event_name]
log.info(f'Setting event for {event_name}')
event.set()
if __name__ == "__main__":
initialize()
processRules = pkg.components.processComponentMap["rules"][
'ProducerConsumer']
componentMap = {}
pipeline_name = processRules['pipelineName']
producers = processRules['producers']
consumers = processRules['consumers']
event_name = pipeline_name + '-event'
pkg.components.processComponentMap['employee-pipeline'] = Pipeline()
pkg.components.processComponentMap[
'employee-pipeline-event'] = threading.Event()
max_worker_count = 0
for k, v in producers.items():
max_worker_count = max_worker_count + v
for k, v in consumers.items():
max_worker_count = max_worker_count + v
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_worker_count) as executor:
for k, v in producers.items():
for id in range(0, v):
log.info(f"submitting producer {k}-{id + 1}")
executor.submit(producer, pipeline_name, k, event_name, id + 1)
for k, v in consumers.items():
for id in range(0, v):
log.info(f"submitting consumer {k}-{id + 1}")
executor.submit(consumer, pipeline_name, k, event_name, id + 1)
#time.sleep(0.1)
pkg.components.processComponentMap[event_name].set()
log.info(f"Max worker count is {max_worker_count}")