def test_import_from_file(self, mock_open, mock_load, mock_isfile):
"""Test process-service file import"""
# set return values
mock_isfile.return_value = True
mock_file = mock.MagicMock(name='service_file')
mock_file.__enter__.return_value = mock_file
mock_open.return_value = mock_file
# create mock process-service class and instance
logger = Logger()
ps_cls = type('ps_cls', (), {'persist': True, 'logger': logger})
ps = mock.Mock(name='ProcessService_instance')
ps.__class__ = ps_cls
# test normal import
mock_load.return_value = ps
ps_ = ProcessService.import_from_file.__func__(ps_cls,
'mock_file_path')
self.assertIs(ps_, ps, 'unexpected process-service instance returned')
mock_open.assert_called_once_with('mock_file_path', 'rb')
mock_load.assert_called_once_with(mock_file)
mock_open.reset_mock()
mock_load.reset_mock()
# test importing instance of incorrect type
mock_load.return_value = None
with self.assertRaises(TypeError):
ProcessService.import_from_file.__func__(ps_cls, 'mock_file_path')
mock_open.reset_mock()
mock_load.reset_mock()
# test import with non-persisting service
ps_cls.persist = False
ps_ = ProcessService.import_from_file.__func__(ps_cls,
'mock_file_path')
self.assertIs(ps_, None,
'unexpected return value for non-persisting service')
Macro does ...(fill in short description here)
Authors:
Your name(s) here
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
from escore import process_manager, Chain, ConfigObject, core_ops
from escore.logger import Logger, LogLevel
logger = Logger()
logger.debug('Now parsing configuration file esk112_parallel_fork_demo.')
# --- minimal analysis information
settings = process_manager.service(ConfigObject)
settings['analysisName'] = 'esk112_parallel_fork_demo'
settings['version'] = 0
# --- now set up the chains and links
ch = Chain('Start')
ch.n_fork = 100
fe = core_ops.ForkExample()
fe.store_key = 'forkstoredemo'
fe.logger.log_level = LogLevel.DEBUG
class Processor(metaclass=ABCMeta):
"""Processor metaclass."""
logger = Logger() # type: Logger
def __init__(self, name: str):
"""Initialize the Processor object."""
super().__init__()
name = name or self.__class__.__name__
self.__name = name # type: str
self.__hash = None # type: int
self.__parent = None
def __str__(self) -> str:
return self.__name
def __repr__(self) -> str:
return '<{klass!s} name={name!s} parent={parent!r} id={id!s}>'.format(
klass=self.__class__.__name__,
name=self.name,
parent=self.__parent,
id=id(self))
def __eq__(self, other: 'Processor') -> bool:
return isinstance(other, type(self)) and self.__name == other.__name
def __hash__(self) -> int:
if self.__hash is None:
self.__hash = hash((type(self), self.__name))
return self.__hash
def _initialize(self):
"""Wrapper to call user implemented initialize."""
self.logger.debug('Initializing link "{link!s}".', link=self)
status = self.initialize()
if status == StatusCode.Success:
self.logger.debug('Successfully initialized link "{link!s}".',
link=self)
return status
@abstractmethod
def initialize(self):
"""Initialization logic for processor."""
raise NotImplementedError
def _execute(self):
"""Wrapper to call user implemented execute."""
self.logger.debug('Executing link "{link!s}".', link=self)
status = self.execute()
if status == StatusCode.Success:
self.logger.debug('Successfully executed link "{link!s}".',
link=self)
return status
@abstractmethod
def execute(self):
"""Execution logic for processor."""
raise NotImplementedError
def _finalize(self):
"""Wrapper to call user implemented finalize."""
self.logger.debug('Finalizing link "{link!s}".', link=self)
status = self.finalize()
if status == StatusCode.Success:
self.logger.debug('Successfully finalized link "{link!s}".',
link=self)
return status
@abstractmethod
def finalize(self):
"""Finalization logic for processor."""
raise NotImplementedError
@property
def name(self) -> str:
"""Get the name of processor.
:return: The name of the processor.
:rtype: str
"""
return self.__name
@property
def parent(self):
"""Get the group parent.
:return: The parent/group processor sequence.
"""
return self.__parent
@parent.setter
def parent(self, the_parent) -> None:
"""Set the group parent.
:param the_parent: The parent/group processor sequence.
"""
self.__parent = None
if the_parent is not None:
# The parent will most likely outlive the processor
# and therefore we do not want keep a strong reference
# to the parent.
self.__parent = proxy(the_parent)
class ProcessService(metaclass=ProcessServiceMeta):
"""Base class for process services."""
logger = Logger()
_persist = False
def __init__(self):
"""Initialize service instance."""
pass
def __str__(self):
"""Get printable specification of service instance."""
return '{0!s} ({1:s})'.format(type(self), hex(id(self)))
@classmethod
def create(cls):
"""Create an instance of this service.
:returns: service instance
:rtype: ProcessService
"""
# create instance and make sure the service is initialized
inst = cls()
ProcessService.__init__(inst)
return inst
def finish(self):
"""Finish current processes.
This function can be implemented by a process-service implementation to
finish running processes and clean up to prepare for a reset of the
process manager. This would typically involve deleting large objects
and closing files and database connections.
"""
pass
@classmethod
def import_from_file(cls, file_path):
"""Import service instance from a Pickle file.
:param str file_path: path of Pickle file
:returns: imported service instance
:rtype: ProcessService
:raises: RuntimeError, TypeError
"""
# check if service can be persisted
if cls.persist:
cls.logger.debug(
'Importing service instance of "{cls!s}" from file "{path}".',
cls=cls,
path=file_path)
else:
cls.logger.debug('Not importing service "{cls!s}".', cls=cls)
return None
# check specified file path
if not os.path.isfile(file_path):
cls.logger.fatal(
'Specified path for importing "{cls!s}" instance is not a file "{path}".',
cls=cls,
path=file_path)
raise RuntimeError(
'Invalid file path specified for importing process service.')
try:
# try to open file and import instance
with open(file_path, 'rb') as inst_file:
inst = pickle.load(inst_file)
except Exception as exc:
# re-raise exeption if import failed
cls.logger.warning(
'Failed to import service instance of "{cls!s}" from file "{path}".',
cls=cls,
path=file_path)
raise exc
# check type of instance
if not isinstance(inst, cls):
cls.logger.fatal(
'Expected to import "{cls!s}" instance, got object of type "{type}".',
cls=cls,
type=type(inst).__name__)
raise TypeError('Incorrect type for imported service object.')
return inst
def persist_in_file(self, file_path):
"""Persist service instance in Pickle file.
:param str file_path: path of Pickle file
"""
# check if service can be persisted
if type(self).persist:
self.logger.debug(
'Persisting service instance "{instance!s}" in file "{path}".',
instance=self,
path=file_path)
else:
self.logger.debug('Not persisting service "{type!s}".',
type=type(self))
return
try:
# try to persist
with open(file_path, 'wb') as inst_file:
pickle.dump(self, inst_file)
except Exception as exc:
# give warning if persisting failed
self.logger.warning(
'Failed to persist service instance "{instance!s}" in file "{path}".',
instance=self,
path=file_path)
self.logger.warning('Caught exception "{exc!s}".', exc=exc)
Authors:
KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""
import datetime
import os
import sys
from escore.logger import Logger
logger = Logger(__name__)
def get_absolute_path(path):
"""Get an absolute path.
First expands ~ if present. Second take care of any . or ..
:param path: path
:returns: the absolute path
"""
return os.path.abspath(os.path.expanduser(path))
def create_file(path, file_name, content=''):
"""Create a file in a given directory.