logger.debug('Now parsing configuration file esk112_parallel_fork_demo.') # --- minimal analysis information settings = process_manager.service(ConfigObject) settings['analysisName'] = 'esk112_parallel_fork_demo' settings['version'] = 0 # --- now set up the chains and links ch = Chain('Start') ch.n_fork = 100 fe = core_ops.ForkExample() fe.store_key = 'forkstoredemo' fe.logger.log_level = LogLevel.DEBUG ch.add(fe) dc = core_ops.ForkDataCollector() dc.keys = [{'key_ds': fe.store_key, 'func': len}] dc.logger.log_level = LogLevel.DEBUG ch.add(dc) ch = Chain('Overview') link = core_ops.PrintDs() link.keys = [fe.store_key] ch.add(link) logger.debug('Done parsing configuration file esk112_parallel_fork_demo.') if __name__ == "__main__":
ds = process_manager.service(DataStore) ds['hello'] = 'world' ds['d'] = {'a': 1, 'b': 2, 'c': 3} ######################################################################################### # --- now set up the chains and links based on configuration flags ch = Chain('Overview') # 1. printdatastore prints an overview of the contents in the datastore # at the state of executing the link. # The overview consists of list of keys in the datastore and and the object types. link = core_ops.PrintDs(name='printer1') # keys are the items for which the contents of the actual item is printed. link.keys = ['hello', 'd'] ch.add(link) # 2. This link will start a python session. # from this session, one can access the datastore and the configobject with: # >>> ds # or # >>> settings # Try to add something to the datastore in this session! # >>> ds['foo'] = 'bar' if not settings['TESTING']: link = core_ops.IPythonEmbed() ch.add(link) # 3. let's see what has been added to the datastore ... link = core_ops.PrintDs(name='printer2') # keys are the items for which the contents of the actual item is printed.
######################################################################################### # --- minimal analysis information settings = process_manager.service(ConfigObject) settings['analysisName'] = 'esk101_helloworld' settings['version'] = 0 ######################################################################################### # --- Analysis values, settings, helper functions, configuration flags. # E.g. define flags turn on or off certain chains with links. # by default all set to false, unless already configured in # configobject or vars() settings['do_hello'] = True settings['n_repeat'] = 2 ######################################################################################### # --- now set up the chains and links based on configuration flags if settings['do_hello']: hello = Chain(name='Hello') link = core_ops.HelloWorld(name='HelloWorld') link.logger.log_level = LogLevel.DEBUG link.repeat = settings['n_repeat'] hello.add(link) ######################################################################################### logger.debug('Done parsing configuration file esk101_helloworld')
######################################################################################### # --- now set up the chains and links, based on configuration flags # --- example loops over the first chain 10 times. if settings['do_example']: # --- a loop is set up in the chain MyChain. # we iterate over the chain until the link RepeatChain is done. # then move on to the next chain (Overview) ch = Chain('MyChain') link = core_ops.HelloWorld(name='HelloWorld') link.logger.log_level = LogLevel.DEBUG ch.add(link) # --- this link sends out a signal to repeat the execution of the chain. # It serves as the 'continue' statement of the loop. # go back to start of the chain until counter reaches 10. repeater = core_ops.RepeatChain() # repeat max of 10 times repeater.maxcount = 10 repeater.logger.log_level = LogLevel.DEBUG ch.add(repeater) # --- print contents of the datastore. # which in this case is empty. overview = Chain('Overview') pds = core_ops.PrintDs(name='End') overview.add(pds)
######################################################################################### # --- now set up the chains and links based on configuration flags # This chain does 'mapping'. (macro B does 'reduction'.) # --- mapper: chain with event looper # this eventlooper link serves as a mapper. # in this example the lines are converted to lower chars, and the first word is selected. if settings['do_map']: ch = Chain("Mapper") looper = core_ops.EventLooper(name='listener') looper.skip_line_beginning_with = ['#'] looper.line_processor_set = [first_word, to_lower] if settings['TESTING']: looper.filename = f.name ch.add(looper) # --- reducer: chain with event looper # this eventlooper link serves as a reducer # in this example the lines are grouped together into unique sets. if settings['do_reduce']: ch = Chain("Reducer") looper = core_ops.EventLooper(name='grouper') # reducer selects all unique lines looper.sort = True looper.unique = True looper.store_key = 'products' if settings['TESTING']: looper.filename = f.name ch.add(looper)