def test_import_from_file(self, mock_open, mock_load, mock_isfile):
"""Test process-service file import"""
# set return values
mock_isfile.return_value = True
mock_file = mock.MagicMock(name='service_file')
mock_file.__enter__.return_value = mock_file
mock_open.return_value = mock_file
# create mock process-service class and instance
logger = Logger()
ps_cls = type('ps_cls', (), {'persist': True, 'logger': logger})
ps = mock.Mock(name='ProcessService_instance')
ps.__class__ = ps_cls
# test normal import
mock_load.return_value = ps
ps_ = ProcessService.import_from_file.__func__(ps_cls,
'mock_file_path')
self.assertIs(ps_, ps, 'unexpected process-service instance returned')
mock_open.assert_called_once_with('mock_file_path', 'rb')
mock_load.assert_called_once_with(mock_file)
mock_open.reset_mock()
mock_load.reset_mock()
# test importing instance of incorrect type
mock_load.return_value = None
with self.assertRaises(TypeError):
ProcessService.import_from_file.__func__(ps_cls, 'mock_file_path')
mock_open.reset_mock()
mock_load.reset_mock()
# test import with non-persisting service
ps_cls.persist = False
ps_ = ProcessService.import_from_file.__func__(ps_cls,
'mock_file_path')
self.assertIs(ps_, None,
'unexpected return value for non-persisting service')
Macro to illustrate the use of multiple chains
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject, Chain
from escore import core_ops
from escore import process_manager
from escore.logger import Logger
logger = Logger()
logger.debug('Now parsing configuration file esk102_multiple_chains')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk102_multiple_chains'
settings['version'] = 0
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
settings['do_chain0'] = True
settings['do_chain1'] = True
Macro to illustrate the use of the Printdatastore link.
Prindatastore prints an overview of the contents in the
datastore at the state of running
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import core_ops, process_manager, ConfigObject, DataStore, Chain
from escore.logger import Logger
logger = Logger()
logger.debug('Now parsing configuration file esk103_printdatastore')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk103_printdatastore'
settings['version'] = 0
#########################################################################################
# --- for this macro, fill the datastore with some dummy information
process_manager.service(DataStore)['hello'] = 'world'
process_manager.service(DataStore)['d'] = {'a': 1, 'b': 2, 'c': 3}
import sys
from pytest import raises
from escore import entry_points
from escore.entry_points import eskapade_bootstrap
from escore.logger import Logger
logger = Logger()
def test_bootstrap_args():
"""Test eskapade_bootstrap arguments handling."""
# no required arguments
with raises(SystemExit) as excinfo:
sys.argv[1:] = []
eskapade_bootstrap()
assert 'the following arguments are required' in str(excinfo.value)
# unrecognized arguments
with raises(SystemExit) as excinfo:
sys.argv[1:] = ['package', 'unrecognized_arg']
eskapade_bootstrap()
assert 'unrecognized arguments' in str(excinfo.value)
# use of Eskapade reserved name
with raises(AttributeError, match='eskapade is reserved by Eskapade'):
sys.argv[1:] = ['eskapade']
eskapade_bootstrap()
# use of Eskapade reserved name
with raises(AttributeError, match='escore is reserved by Eskapade'):
sys.argv[1:] = ['escore']
eskapade_bootstrap()
class ProcessService(metaclass=ProcessServiceMeta):
"""Base class for process services."""
logger = Logger()
_persist = False
def __init__(self):
"""Initialize service instance."""
pass
def __str__(self):
"""Get printable specification of service instance."""
return '{0!s} ({1:s})'.format(type(self), hex(id(self)))
@classmethod
def create(cls):
"""Create an instance of this service.
:returns: service instance
:rtype: ProcessService
"""
# create instance and make sure the service is initialized
inst = cls()
ProcessService.__init__(inst)
return inst
def finish(self):
"""Finish current processes.
This function can be implemented by a process-service implementation to
finish running processes and clean up to prepare for a reset of the
process manager. This would typically involve deleting large objects
and closing files and database connections.
"""
pass
@classmethod
def import_from_file(cls, file_path):
"""Import service instance from a Pickle file.
:param str file_path: path of Pickle file
:returns: imported service instance
:rtype: ProcessService
:raises: RuntimeError, TypeError
"""
# check if service can be persisted
if cls.persist:
cls.logger.debug(
'Importing service instance of "{cls!s}" from file "{path}".',
cls=cls,
path=file_path)
else:
cls.logger.debug('Not importing service "{cls!s}".', cls=cls)
return None
# check specified file path
if not os.path.isfile(file_path):
cls.logger.fatal(
'Specified path for importing "{cls!s}" instance is not a file "{path}".',
cls=cls,
path=file_path)
raise RuntimeError(
'Invalid file path specified for importing process service.')
try:
# try to open file and import instance
with open(file_path, 'rb') as inst_file:
inst = pickle.load(inst_file)
except Exception as exc:
# re-raise exeption if import failed
cls.logger.warning(
'Failed to import service instance of "{cls!s}" from file "{path}".',
cls=cls,
path=file_path)
raise exc
# check type of instance
if not isinstance(inst, cls):
cls.logger.fatal(
'Expected to import "{cls!s}" instance, got object of type "{type}".',
cls=cls,
type=type(inst).__name__)
raise TypeError('Incorrect type for imported service object.')
return inst
def persist_in_file(self, file_path):
"""Persist service instance in Pickle file.
:param str file_path: path of Pickle file
"""
# check if service can be persisted
if type(self).persist:
self.logger.debug(
'Persisting service instance "{instance!s}" in file "{path}".',
instance=self,
path=file_path)
else:
self.logger.debug('Not persisting service "{type!s}".',
type=type(self))
return
try:
# try to persist
with open(file_path, 'wb') as inst_file:
pickle.dump(self, inst_file)
except Exception as exc:
# give warning if persisting failed
self.logger.warning(
'Failed to persist service instance "{instance!s}" in file "{path}".',
instance=self,
path=file_path)
self.logger.warning('Caught exception "{exc!s}".', exc=exc)
Follow-up example in macro: esk209_readdata_itr.py
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject, Chain
from escore import core_ops
from escore import process_manager
from escore.logger import Logger, LogLevel
logger = Logger()
logger.debug('Now parsing configuration file esk107_chain_looper')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk107_chain_looper'
settings['version'] = 0
#########################################################################################
# --- Analysis configuration flags.
# E.g. use these flags turn on or off certain chains with links.
# by default all set to false, unless already configured in
# configobject or vars()
Macro to illustrate the use of flags set from the command line.
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject, Chain
from escore import core_ops
from escore import process_manager
from escore.logger import Logger
logger = Logger()
logger.debug('Now parsing configuration file esk106_cmdline_options')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk106_cmdline_options'
settings['version'] = 0
msg = r"""
The two flags below control whether chains are turned on or off. (default=on)
from the cmd line, control these with:
-c do_chain0=False -c do_chain1=False
Macro illustrates how to load an external datastore from file
Authors:
KMPG AA&BD team
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
import shutil
from escore import process_manager, Chain, ConfigObject, DataStore, core_ops
from escore.logger import Logger, LogLevel
from escore.core import persistence
logger = Logger()
logger.debug('Now parsing configuration file esk111_load_datastore_from_file.')
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk111_load_datastore_from_file'
settings['version'] = 0
ds = process_manager.service(DataStore)
ds['number'] = 1
file_path = persistence.io_path('proc_service_data', 'temp_datastore.pkl')
ds.persist_in_file(file_path)
# --- update the number
Macro illustrates how to persist the datastore
and configuration object after each chain.
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject
from escore import process_manager, resources
from escore.logger import Logger
logger = Logger()
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
# turning on this flag, the datastore and configuration are not written out to disk
# at the end of the program.
settings = process_manager.service(ConfigObject)
settings['storeResultsEachChain'] = True
msg = r"""
The global flag settings['storeResultsEachChain'] (default = False)
controls persistence of the run-process services after the execution of
a chain. By default, these objects are only stored after the last
Macro to say hello to the world with Eskapade!
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject, Chain
from escore import core_ops
from escore import process_manager
from escore.logger import Logger, LogLevel
logger = Logger()
logger.debug('Now parsing configuration file esk101_helloworld')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk101_helloworld'
settings['version'] = 0
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
# E.g. define flags turn on or off certain chains with links.
# by default all set to false, unless already configured in
Macro serves as input to other three esk105 example macros.
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject, Chain
from escore import core_ops
from escore import process_manager
from escore.logger import Logger
logger = Logger()
logger.debug('Now parsing configuration file esk105_datastore_pickling.')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk105_datastore_pickling'
settings['version'] = 0
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
msg = r"""
The setup consists of three simple chains that add progressively more information to the datastore.
class Processor(metaclass=ABCMeta):
"""Processor metaclass."""
logger = Logger() # type: Logger
def __init__(self, name: str):
"""Initialize the Processor object."""
super().__init__()
name = name or self.__class__.__name__
self.__name = name # type: str
self.__hash = None # type: int
self.__parent = None
def __str__(self) -> str:
return self.__name
def __repr__(self) -> str:
return '<{klass!s} name={name!s} parent={parent!r} id={id!s}>'.format(
klass=self.__class__.__name__,
name=self.name,
parent=self.__parent,
id=id(self))
def __eq__(self, other: 'Processor') -> bool:
return isinstance(other, type(self)) and self.__name == other.__name
def __hash__(self) -> int:
if self.__hash is None:
self.__hash = hash((type(self), self.__name))
return self.__hash
def _initialize(self):
"""Wrapper to call user implemented initialize."""
self.logger.debug('Initializing link "{link!s}".', link=self)
status = self.initialize()
if status == StatusCode.Success:
self.logger.debug('Successfully initialized link "{link!s}".',
link=self)
return status
@abstractmethod
def initialize(self):
"""Initialization logic for processor."""
raise NotImplementedError
def _execute(self):
"""Wrapper to call user implemented execute."""
self.logger.debug('Executing link "{link!s}".', link=self)
status = self.execute()
if status == StatusCode.Success:
self.logger.debug('Successfully executed link "{link!s}".',
link=self)
return status
@abstractmethod
def execute(self):
"""Execution logic for processor."""
raise NotImplementedError
def _finalize(self):
"""Wrapper to call user implemented finalize."""
self.logger.debug('Finalizing link "{link!s}".', link=self)
status = self.finalize()
if status == StatusCode.Success:
self.logger.debug('Successfully finalized link "{link!s}".',
link=self)
return status
@abstractmethod
def finalize(self):
"""Finalization logic for processor."""
raise NotImplementedError
@property
def name(self) -> str:
"""Get the name of processor.
:return: The name of the processor.
:rtype: str
"""
return self.__name
@property
def parent(self):
"""Get the group parent.
:return: The parent/group processor sequence.
"""
return self.__parent
@parent.setter
def parent(self, the_parent) -> None:
"""Set the group parent.
:param the_parent: The parent/group processor sequence.
"""
self.__parent = None
if the_parent is not None:
# The parent will most likely outlive the processor
# and therefore we do not want keep a strong reference
# to the parent.
self.__parent = proxy(the_parent)
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
import datetime
import os
import sys
from escore.logger import Logger
logger = Logger(__name__)
def get_absolute_path(path):
"""Get an absolute path.
First expands ~ if present. Second take care of any . or ..
:param path: path
:returns: the absolute path
"""
return os.path.abspath(os.path.expanduser(path))
def create_file(path, file_name, content=''):
"""Create a file in a given directory.
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject, Chain
from escore import DataStore
from escore import core_ops
from escore import process_manager
from escore.logger import Logger
logger = Logger()
logger.debug('Now parsing configuration file esk109_debugging_tips')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk109_debugging_tips'
settings['version'] = 0
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
msg = r"""
Macro to illustrate how to control the contents of the datastore
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject, Chain
from escore import core_ops
from escore import process_manager
from escore.logger import Logger
logger = Logger()
logger.debug('Now parsing configuration file esk104_basic_datastore_operations.')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk104_basic_datastore_operations'
settings['version'] = 0
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
# some dummy information to use in this macro
f = {'hello': 'world', 'v': [3, 1, 4, 1, 5], 'n_favorite': 7}
Macro does ...(fill in short description here)
Authors:
Your name(s) here
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import process_manager, Chain, ConfigObject, core_ops
from escore.logger import Logger, LogLevel
logger = Logger()
logger.debug('Now parsing configuration file esk112_parallel_fork_demo.')
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk112_parallel_fork_demo'
settings['version'] = 0
# --- now set up the chains and links
ch = Chain('Start')
ch.n_fork = 100
fe = core_ops.ForkExample()
fe.store_key = 'forkstoredemo'
fe.logger.log_level = LogLevel.DEBUG
Description:
Macro to demo how to run eskapade with code profiling turned on
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import ConfigObject
from escore import process_manager
from escore.logger import Logger
logger = Logger()
logger.debug('Now parsing configuration file esk110_code_profiling.')
#########################################################################################
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk110_code_profiling'
settings['version'] = 0
#########################################################################################
# --- Analysis values, settings, helper functions, configuration flags.
msg = r"""