logger.debug('Now parsing configuration file esk112_parallel_fork_demo.')
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk112_parallel_fork_demo'
settings['version'] = 0
# --- now set up the chains and links
ch = Chain('Start')
ch.n_fork = 100
fe = core_ops.ForkExample()
fe.store_key = 'forkstoredemo'
fe.logger.log_level = LogLevel.DEBUG
ch.add(fe)
dc = core_ops.ForkDataCollector()
dc.keys = [{'key_ds': fe.store_key, 'func': len}]
dc.logger.log_level = LogLevel.DEBUG
ch.add(dc)
ch = Chain('Overview')
link = core_ops.PrintDs()
link.keys = [fe.store_key]
ch.add(link)
logger.debug('Done parsing configuration file esk112_parallel_fork_demo.')
if __name__ == "__main__":
ds = process_manager.service(DataStore)
ds['hello'] = 'world'
ds['d'] = {'a': 1, 'b': 2, 'c': 3}
#########################################################################################
# --- now set up the chains and links based on configuration flags
ch = Chain('Overview')
# 1. printdatastore prints an overview of the contents in the datastore
# at the state of executing the link.
# The overview consists of list of keys in the datastore and and the object types.
link = core_ops.PrintDs(name='printer1')
# keys are the items for which the contents of the actual item is printed.
link.keys = ['hello', 'd']
ch.add(link)
# 2. This link will start a python session.
# from this session, one can access the datastore and the configobject with:
# >>> ds
# or
# >>> settings
# Try to add something to the datastore in this session!
# >>> ds['foo'] = 'bar'
if not settings['TESTING']:
link = core_ops.IPythonEmbed()
ch.add(link)
# 3. let's see what has been added to the datastore ...
link = core_ops.PrintDs(name='printer2')
# keys are the items for which the contents of the actual item is printed.
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk101_helloworld'
settings['version'] = 0
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
# E.g. define flags turn on or off certain chains with links.
# by default all set to false, unless already configured in
# configobject or vars()
settings['do_hello'] = True
settings['n_repeat'] = 2
#########################################################################################
# --- now set up the chains and links based on configuration flags
if settings['do_hello']:
hello = Chain(name='Hello')
link = core_ops.HelloWorld(name='HelloWorld')
link.logger.log_level = LogLevel.DEBUG
link.repeat = settings['n_repeat']
hello.add(link)
#########################################################################################
logger.debug('Done parsing configuration file esk101_helloworld')
#########################################################################################
# --- now set up the chains and links, based on configuration flags
# --- example loops over the first chain 10 times.
if settings['do_example']:
# --- a loop is set up in the chain MyChain.
# we iterate over the chain until the link RepeatChain is done.
# then move on to the next chain (Overview)
ch = Chain('MyChain')
link = core_ops.HelloWorld(name='HelloWorld')
link.logger.log_level = LogLevel.DEBUG
ch.add(link)
# --- this link sends out a signal to repeat the execution of the chain.
# It serves as the 'continue' statement of the loop.
# go back to start of the chain until counter reaches 10.
repeater = core_ops.RepeatChain()
# repeat max of 10 times
repeater.maxcount = 10
repeater.logger.log_level = LogLevel.DEBUG
ch.add(repeater)
# --- print contents of the datastore.
# which in this case is empty.
overview = Chain('Overview')
pds = core_ops.PrintDs(name='End')
overview.add(pds)
#########################################################################################
# --- now set up the chains and links based on configuration flags
# This chain does 'mapping'. (macro B does 'reduction'.)
# --- mapper: chain with event looper
# this eventlooper link serves as a mapper.
# in this example the lines are converted to lower chars, and the first word is selected.
if settings['do_map']:
ch = Chain("Mapper")
looper = core_ops.EventLooper(name='listener')
looper.skip_line_beginning_with = ['#']
looper.line_processor_set = [first_word, to_lower]
if settings['TESTING']:
looper.filename = f.name
ch.add(looper)
# --- reducer: chain with event looper
# this eventlooper link serves as a reducer
# in this example the lines are grouped together into unique sets.
if settings['do_reduce']:
ch = Chain("Reducer")
looper = core_ops.EventLooper(name='grouper')
# reducer selects all unique lines
looper.sort = True
looper.unique = True
looper.store_key = 'products'
if settings['TESTING']:
looper.filename = f.name
ch.add(looper)