# --- minimal analysis information settings = process_manager.service(ConfigObject) settings['analysisName'] = 'esk111_load_datastore_from_file' settings['version'] = 0 ds = process_manager.service(DataStore) ds['number'] = 1 file_path = persistence.io_path('proc_service_data', 'temp_datastore.pkl') ds.persist_in_file(file_path) # --- update the number ds['number'] = 2 # --- Reload from the pickle file with: # >>> ds = DataStore.import_from_file(file_path) # --- now set up the chains and links ch = Chain('Start') link = core_ops.ImportDataStore(name='importds', path=file_path) link.logger.log_level = LogLevel.DEBUG ch.add(link) link = core_ops.PrintDs() link.keys = ['number'] ch.add(link) logger.debug( 'Done parsing configuration file esk111_load_datastore_from_file.')
{conf_path} """ logger.info(msg, data_path=settings['resultsDir'] + '/' + settings['analysisName'] + '/data/v0/', conf_path=settings['resultsDir'] + '/' + settings['analysisName'] + '/config/v0/') # dummy information used in this macro, added to each chain below. f = {'hello': 'world', 'v': [3, 1, 4, 1, 5], 'n_favorite': 7} g = {'a': 1, 'b': 2, 'c': 3} h = [2, 7] ######################################################################################### # --- now set up the chains and links based on configuration flags ######### # chain 1 ch = Chain('chain1') # the link ToDsDict adds objects to the datastore at link execution. link = core_ops.ToDsDict(name='intods_1') link.store_key = 'f' link.obj = f ch.add(link) # print contents of datastore link = core_ops.PrintDs() ch.add(link) ######### # chain 2 ch = Chain('chain2')
def first_word(x): """Take first word.""" return x.split()[0] ######################################################################################### # --- now set up the chains and links based on configuration flags # This chain does 'mapping'. (macro B does 'reduction'.) # --- mapper: chain with event looper # this eventlooper link serves as a mapper. # in this example the lines are converted to lower chars, and the first word is selected. if settings['do_map']: ch = Chain("Mapper") looper = core_ops.EventLooper(name='listener') looper.skip_line_beginning_with = ['#'] looper.line_processor_set = [first_word, to_lower] if settings['TESTING']: looper.filename = f.name ch.add(looper) # --- reducer: chain with event looper # this eventlooper link serves as a reducer # in this example the lines are grouped together into unique sets. if settings['do_reduce']: ch = Chain("Reducer") looper = core_ops.EventLooper(name='grouper') # reducer selects all unique lines looper.sort = True