def test_esk105a(self):
self.eskapade_run(resources.tutorial('esk105_A_dont_store_results.py'))
settings = process_manager.service(ConfigObject)
path = settings['resultsDir'] + '/' + settings['analysisName']
self.assertFalse(os.path.exists(path))
def test_esk101(self):
self.eskapade_run(resources.tutorial('esk101_helloworld.py'))
settings = process_manager.service(ConfigObject)
self.assertTrue(settings['do_hello'])
self.assertEqual(2, settings['n_repeat'])
def test_esk104(self):
self.eskapade_run(
resources.tutorial('esk104_basic_datastore_operations.py'))
ds = process_manager.service(DataStore)
self.assertEqual(1, len(ds))
self.assertEqual(1, ds['a'])
def test_esk108reduce(self):
settings = process_manager.service(ConfigObject)
settings['TESTING'] = True
self.eskapade_run(resources.tutorial('esk108_reduce.py'))
ds = process_manager.service(DataStore)
self.assertEqual(20, ds['n_products'])
def test_esk107(self):
self.eskapade_run(resources.tutorial('esk107_chain_looper.py'))
ds = process_manager.service(DataStore)
# chain is repeated 10 times, with nothing put in datastore
self.assertEqual(0, len(ds))
self.assertEqual(10, list(list(process_manager)[0])[1].maxcount)
def test_esk103(self):
self.eskapade_run(resources.tutorial('esk103_printdatastore.py'))
ds = process_manager.service(DataStore)
self.assertEqual('world', ds['hello'])
self.assertEqual(1, ds['d']['a'])
self.assertEqual(2, ds['d']['b'])
self.assertEqual(3, ds['d']['c'])
def test_esk102(self):
self.eskapade_run(resources.tutorial('esk102_multiple_chains.py'))
settings = process_manager.service(ConfigObject)
self.assertTrue(settings['do_chain0'])
self.assertTrue(settings['do_chain1'])
self.assertTrue(settings['do_chain2'])
self.assertEqual(3, len(process_manager))
def test_esk109(self):
settings = process_manager.service(ConfigObject)
# this flag turns off python embed link
settings['TESTING'] = True
self.eskapade_run(resources.tutorial('esk109_debugging_tips.py'),
StatusCode.Failure)
self.assertTrue(isinstance(list(list(process_manager)[0])[3], Break))
def test_esk106_script(self, mock_argv):
"""Test Eskapade run with esk106 macro from script"""
# get file paths
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk106_cmdline_options'
settings_ = settings.copy()
macro_path = resources.tutorial('esk106_cmdline_options.py')
# mock command-line arguments
args = []
mock_argv.__getitem__ = lambda s, k: args.__getitem__(k)
# base settings
args_ = [macro_path, '-LDEBUG', '--batch-mode']
settings_['macro'] = macro_path
settings_['logLevel'] = LogLevel.DEBUG
settings_['batchMode'] = True
def do_run(name, args, args_, settings_, add_args, add_settings,
chains):
# set arguments
args.clear()
args += args_ + add_args
settings = settings_.copy()
settings.update(add_settings)
# run Eskapade
process_manager.reset()
entry_points.eskapade_run()
settings_run = process_manager.service(ConfigObject)
# check results
self.assertListEqual(
[c.name for c in process_manager.chains], chains,
'unexpected chain names in "{}" test'.format(name))
self.assertDictEqual(
settings_run, settings,
'unexpected settings in "{}" test'.format(name))
# run both chains
do_run(
'both chains', args, args_, settings_,
['--store-all', '-cdo_chain0=True', '-cdo_chain1=True'],
dict(storeResultsEachChain=True, do_chain0=True,
do_chain1=True), ['Chain0', 'Chain1'])
# run only last chain by skipping the first
do_run('skip first', args, args_, settings_,
['-bChain1', '-cdo_chain0=True', '-cdo_chain1=True'],
dict(beginWithChain='Chain1', do_chain0=True,
do_chain1=True), ['Chain0', 'Chain1'])
# run only last chain by not defining the first
do_run('no first', args, args_, settings_,
['-cdo_chain0=False', '-cdo_chain1=True'],
dict(do_chain0=False, do_chain1=True), ['Chain1'])
def test_esk110(self):
self.eskapade_run(resources.tutorial('esk110_code_profiling.py'))
settings = process_manager.service(ConfigObject)
ds = process_manager.service(DataStore)
self.assertEqual(0, len(process_manager))
self.assertEqual(0, len(ds))
self.assertTrue('doCodeProfiling' in settings)
self.assertEqual('cumulative', settings['doCodeProfiling'])
def test_esk106(self):
settings = process_manager.service(ConfigObject)
# fake a setting from the cmd-line. picked up in the macro
settings['do_chain0'] = False
self.eskapade_run(resources.tutorial('esk106_cmdline_options.py'))
settings = process_manager.service(ConfigObject)
self.assertEqual(1, len(process_manager))
self.assertEqual('Chain1', list(process_manager)[0].name)
self.assertEqual(False, settings.get('do_chain0', True))
self.assertEqual(True, settings.get('do_chain1', True))
self.assertEqual('Universe', list(list(process_manager)[0])[0].hello)
def test_esk105bc(self):
self.eskapade_run(resources.tutorial('esk105_B_store_each_chain.py'))
settings = process_manager.service(ConfigObject)
# results of all three chains have been persisted
path = '{0:s}/{1:s}/proc_service_data/v0/_chain{{:d}}/{2:s}.pkl'.format(
settings['resultsDir'], settings['analysisName'], str(DataStore))
for path_it in range(1, 4):
self.assertTrue(os.path.exists(path.format(path_it)))
execution.reset_eskapade()
self.eskapade_run(resources.tutorial('esk105_C_begin_at_chain3.py'))
ds = process_manager.service(DataStore)
# object from all three chains are present
self.assertTrue('f' in ds)
self.assertTrue('g' in ds)
self.assertTrue('h' in ds)
self.assertEqual(3, len(ds))
self.assertEqual(7, ds['f']['n_favorite'])
self.assertEqual(1, ds['g']['a'])
self.assertEqual(7, ds['h'][1])
Macro to illustrate how input lines can be read in,
processed, and reprinted. E.g. for use in map reduce application.
Use in combination with: esk108_map
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject
from escore import process_manager, resources
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
# turning on this flag, the datastore and configuration are not written out to disk
# at the end of the program.
settings = process_manager.service(ConfigObject)
settings['do_reduce'] = True
settings['doNotStoreResults'] = True
#########################################################################################
# --- now parse the follow-up macro
# the flag doNotStoreResults is picked up when parsing the following macro
process_manager.execute_macro(resources.tutorial('esk108_eventlooper.py'))
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
# turning on this flag, the datastore and configuration are not written out to disk
# at the end of the program.
settings = process_manager.service(ConfigObject)
settings['storeResultsEachChain'] = True
msg = r"""
The global flag settings['storeResultsEachChain'] (default = False)
controls persistence of the run-process services after the execution of
a chain. By default, these objects are only stored after the last
chain.
From the command line, this flag is set with option --store-all.
Alternatively, the option --store-one="chain_name" can be used to store
the results of a single chain. The with the option --store-none, no
results are stored.
"""
logger.info(msg)
#########################################################################################
# --- now parse the follow-up macro
# the flag doNotStoreResults is picked up when parsing the following macro
process_manager.execute_macro(
resources.tutorial('esk105_datastore_pickling.py'))
def test_esk108map(self):
settings = process_manager.service(ConfigObject)
settings['TESTING'] = True
self.eskapade_run(resources.tutorial('esk108_map.py'))
# and does so by reading in the datastore written out after chain 2.
settings = process_manager.service(ConfigObject)
settings['beginWithChain'] = 'chain3'
msg = r"""
--> Make sure to run this macro after running macro
esk105_B_store_each_chain.py
By default, the process manager starts the execution of the first
configured chain and end with the last one. If the run-process services
are written out after the execution of each chain (--store-all), it is
possible to execute one particular chain. This is done by picking up
the services written out by the previous chain. This is a nice feature
for debugging and developing purposes.
From the command line use the options:
-b BEGIN_WITH_CHAIN
-e END_WITH_CHAIN
-s SINGLE_CHAIN
"""
logger.info(msg)
#########################################################################################
# --- now parse the follow-up macro
# the flag doNotStoreResults is picked up when parsing the following macro
process_manager.execute_macro(resources.tutorial('esk105_datastore_pickling.py'))