class Quirk(abc.ABC):
"""Base class for sourdough quirks (mixin-approximations).
Args:
library (ClassVar[sourdough.Library]): related Library instance that
will store concrete subclasses and allow runtime construction and
instancing of those stored subclasses.
Namespaces: library, bases
"""
library: ClassVar[sourdough.Library] = sourdough.Library(
contents={'base': Base})
""" Initialization Methods """
def __init_subclass__(cls, **kwargs):
"""Adds 'cls' to 'library' if it is a concrete class."""
super().__init_subclass__(**kwargs)
# Creates a snakecase key of the class name.
key = sourdough.tools.snakify(cls.__name__)
# Adds base classes to 'bases' using 'key'.
if not hasattr(Base.bases, 'quirk'):
setattr(Base.bases, 'quirks', cls)
setattr(cls, 'bases', Base.bases)
setattr(cls.bases, key, cls)
# Adds concrete subclasses to 'library' using 'key'.
if not abc.ABC in cls.__bases__:
cls.library[key] = cls
class Stage(sourdough.quirks.Base, sourdough.quirks.Needy, abc.ABC):
"""Creates a sourdough object.
Args:
needs (ClassVar[Union[Sequence[str], str]]): attributes needed from
another instance for some method within a subclass. Defaults to an
empty list.
library (ClassVar[Library]): related Library instance that will store
subclasses and allow runtime construction and instancing of those
stored subclasses.
"""
needs: ClassVar[Union[Sequence[str], str]] = []
library: ClassVar[sourdough.Library] = sourdough.Library()
""" Class Methods """
@classmethod
def create(cls, **kwargs) -> object:
"""[summary]
Raises:
ValueError: [description]
Returns:
object: [description]
"""
needs = list(more_itertools.always_iterable(cls.needs))
method = getattr(cls, f'from_{needs[0]}')
for need in needs:
if need not in kwargs:
raise ValueError(
f'The create method must include a {need} argument')
return method(**kwargs)
def __init_subclass__(cls, **kwargs):
"""Adds 'cls' to 'library' if it is a concrete class."""
super().__init_subclass__(**kwargs)
# Creates a snakecase key of the class name.
key = sourdough.tools.snakify(cls.__name__)
if Base in cls.__bases__:
# Adds this class to 'bases' using 'key'.
setattr(cls.bases, key, cls)
# Creates a library on this class base for storing subclasses.
cls.library = sourdough.Library()
# Adds concrete subclasses to 'library' using 'key'.
if not abc.ABC in cls.__bases__:
cls.library[key] = cls
class Settings(sourdough.quirks.Base, sourdough.Configuration):
"""Loads and stores configuration settings for a Project.
Args:
contents (Union[str, pathlib.Path, Mapping[str, Mapping[str, Any]]]): a
dict, a str file path to a file with settings, or a pathlib Path to
a file with settings. Defaults to en empty dict.
infer_types (bool): whether values in 'contents' are converted to other
datatypes (True) or left alone (False). If 'contents' was imported
from an .ini file, a False value will leave all values as strings.
Defaults to True.
defaults (Mapping[str, Mapping[str]]): any default options that should
be used when a user does not provide the corresponding options in
their configuration settings. Defaults to a dict with 'general'
and 'files' section listed in the class annotations.
skip (Sequence[str]): names of suffixes to skip when constructing nodes
for a sourdough project.
library (ClassVar[Library]): related Library instance that will store
subclasses and allow runtime construction and instancing of those
stored subclasses.
"""
contents: Union[str, pathlib.Path,
Mapping[str, Mapping[str, Any]]] = (dataclasses.field(
default_factory=dict))
infer_types: bool = True
defaults: Mapping[str, Mapping[str, Any]] = dataclasses.field(
default_factory=lambda: {
'general': {
'verbose': False,
'parallelize': False,
'conserve_memery': False
},
'files': {
'source_format': 'csv',
'interim_format': 'csv',
'final_format': 'csv',
'file_encoding': 'windows-1252'
},
'sourdough': {
'default_design': 'pipeline',
'default_workflow': 'graph'
}
})
skip: Sequence[str] = dataclasses.field(
default_factory=lambda:
['general', 'files', 'sourdough', 'parameters'])
library: ClassVar[sourdough.Library] = sourdough.Library()
class Base(abc.ABC):
"""Abstract base class for automatic registration of subclasses.
Even though not technically a Quirk subclass (because Quirk inherits from
Base), it should be used in the same manner. Base is used throughout the
project subpackage as a Quirk to automatically store base subclasses.
Any non-abstract subclass will automatically store itself in the class
attribute 'library' using the snakecase name of the class as the key.
Any direct subclass will automatically store itself in the class attribute
'bases' using the snakecase name of the class as the key.
Args:
bases (ClassVar[sourdough.Library]): related Library instance that will
store direct subclasses (those with Base in their '__bases__'
attribute) and allow runtime construction and instancing of those
stored subclasses.
Attributes:
library (ClassVar[sourdough.Library]): related Library instance that
will store concrete subclasses and allow runtime construction and
instancing of those stored subclasses. 'library' is automatically
created when a direct Base subclass (Base is in its '__bases__') is
instanced.
Namespaces: library, bases, and __init_subclass__.
"""
bases: ClassVar[sourdough.Library] = sourdough.Library()
""" Initialization Methods """
def __init_subclass__(cls, **kwargs):
"""Adds 'cls' to 'library' if it is a concrete class."""
super().__init_subclass__(**kwargs)
# Creates a snakecase key of the class name.
key = sourdough.tools.snakify(cls.__name__)
if Base in cls.__bases__:
# Adds this class to 'bases' using 'key'.
setattr(cls.bases, key, cls)
# Creates a library on this class base for storing subclasses.
cls.library = sourdough.Library()
# Adds concrete subclasses to 'library' using 'key'.
if not abc.ABC in cls.__bases__:
cls.library[key] = cls
class Validator(sourdough.quirks.Quirk):
"""Mixin for calling validation methods
Args:
validations (List[str]): a list of attributes that need validating.
Each item in 'validations' should have a corresponding method named
f'_validate_{name}' or match a key in 'converters'. Defaults to an
empty list.
converters (sourdough.Library):
"""
validations: ClassVar[Sequence[str]] = []
converters: ClassVar[sourdough.Library] = sourdough.Library()
""" Public Methods """
def initialize_converter(
self, name: str, converter: Union[str, Converter,
Type[Converter]]) -> Converter:
"""[summary]
Args:
converter (Union[Converter, Type[Converter]]): [description]
Returns:
Converter: [description]
"""
if isinstance(converter, str):
try:
converter = self.converters[name]
except KeyError:
raise KeyError(
f'No local or stored type validator exists for {name}')
if not isinstance(converter, Converter):
converter = converter()
self.converters[name] = converter
return converter
def validate(self, validations: Sequence[str] = None) -> None:
"""Validates or converts stored attributes.
Args:
validations (List[str]): a list of attributes that need validating.
Each item in 'validations' should have a corresponding method
named f'_validate_{name}' or match a key in 'converters'. If not
passed, the 'validations' attribute will be used instead.
Defaults to None.
"""
if validations is None:
validations = self.validations
# Calls validation methods based on names listed in 'validations'.
for name in validations:
if hasattr(self, f'_validate_{name}'):
kwargs = {name: getattr(self, name)}
validated = getattr(self, f'_validate_{name}')(**kwargs)
else:
converter = self.initialize_converter(name=name,
converter=name)
try:
validated = converter.validate(item=getattr(self, name),
instance=self)
except AttributeError:
validated = getattr(self, name)
setattr(self, name, validated)
return self
class Component(sourdough.quirks.Base, sourdough.quirks.Element, abc.ABC):
"""Base class for parts of a sourdough Workflow.
Args:
name (str): designates the name of a class instance that is used for
internal referencing throughout sourdough. For example, if a
sourdough instance needs settings from a Configuration instance,
'name' should match the appropriate section name in a Configuration
instance. Defaults to None.
contents (Any): stored item(s) for use by a Component subclass instance.
library (ClassVar[Library]): related Library instance that will store
subclasses and allow runtime construction and instancing of those
stored subclasses.
"""
name: str = None
contents: Any = None
iterations: Union[int, str] = 1
parameters: Mapping[Any, Any] = dataclasses.field(default_factory=dict)
parallel: ClassVar[bool] = False
library: ClassVar[sourdough.Library] = sourdough.Library()
""" Public Class Methods """
@classmethod
def create(cls, source: str, **kwargs) -> Component:
"""[summary]
Args:
source (str): [description]
Raises:
NotImplementedError: [description]
Returns:
Component: [description]
"""
if (isinstance(source, str)
or (isinstance(source, Sequence) and all(source, str))):
return cls.from_name(name=source, **kwargs)
elif isinstance(source, cls.bases.Stage):
return
@classmethod
def from_outline(cls, name: str, outline: sourdough.project.Outline,
**kwargs) -> Component:
"""[summary]
Args:
name (str): [description]
section (str): [description]
outline (sourdough.project.Outline): [description]
Returns:
Component: [description]
"""
if name in outline.initialization:
parameters = outline.initialization[name]
parameters.update(kwargs)
else:
parameters = kwargs
component = cls.library.borrow(names=[name, outline.designs[name]])
instance = component(name=name, **parameters)
return instance
""" Public Methods """
def execute(self, data: Any, **kwargs) -> Any:
"""[summary]
Args:
data (Any): [description]
Returns:
Any: [description]
"""
if self.iterations in ['infinite']:
while True:
data = self.implement(data=data, **kwargs)
else:
for iteration in range(self.iterations):
data = self.implement(data=data, **kwargs)
return data
def implement(self, data: Any, **kwargs) -> Any:
"""[summary]
Args:
data (Any): [description]
Returns:
Any: [description]
"""
if self.parameters:
parameters = self.parameters
parameters.update(kwargs)
else:
parameters = kwargs
if self.contents not in [None, 'None', 'none']:
data = self.contents.implement(data=data, **parameters)
return data
""" Private Methods """
def _add_attributes(self, attributes: Dict[Any, Any]) -> None:
"""[summary]
Args:
attributes (Dict[Any, Any]): [description]
Returns:
[type]: [description]
"""
for key, value in attributes.items():
setattr(self, key, value)
return self
class Manager(sourdough.quirks.Base, sourdough.quirks.Element):
"""
Args:
name (str): designates the name of a class instance that is used for
internal referencing throughout sourdough. For example, if a
sourdough instance needs settings from a Configuration instance,
'name' should match the appropriate section name in a Configuration
instance. Defaults to None.
library (ClassVar[Library]): related Library instance that will store
subclasses and allow runtime construction and instancing of those
stored subclasses.
"""
name: str = None
contents: Any = None
workflow: sourdough.Structure = None
needs: ClassVar[Union[Sequence[str], str]] = ['outline', 'name']
library: ClassVar[sourdough.Library] = sourdough.Library()
""" Public Class Methods """
@classmethod
def create(cls, **kwargs) -> object:
"""[summary]
Raises:
ValueError: [description]
Returns:
object: [description]
"""
needs = list(more_itertools.always_iterable(cls.needs))
method = getattr(cls, f'from_{needs[0]}')
for need in needs:
if need not in kwargs:
raise ValueError(
f'The create method must include a {need} argument')
return method(**kwargs)
@classmethod
def from_outline(cls, outline: Stage, name: str, **kwargs) -> Manager:
"""[summary]
Args:
outline (Stage): [description]
name (str): [description]
Returns:
Manager: [description]
"""
names = [name]
if name in outline.designs:
names.append(outline.design[name])
manager = cls.library.borrow(names=names)
structure = cls.bases.stage.library.borrow(names='workflow')
workflow = structure.create(outline=outline, name=name)
return manager(name=name, workflow=workflow, **kwargs)
""" Public Methods """
def execute(self, data: Any) -> Any:
"""[summary]
Args:
data (Any): [description]
Returns:
Any: [description]
"""
for node in self.workflow:
data = node.execute(data=data)
return data
class Workflow(base.Stage, sourdough.Graph):
"""Stores lightweight workflow and corresponding components.
Args:
contents (Dict[str, List[str]]): an adjacency list where the keys are
the names of nodes and the values are names of nodes which the key
is connected to. Defaults to an empty dict.
default (Any): default value to use when a key is missing and a new
one is automatically corrected. Defaults to an empty list.
components (sourdough.Library): stores Component instances that
correspond to nodes in 'contents'. Defaults to an empty Library.
needs (ClassVar[Union[Sequence[str], str]]): attributes needed from
another instance for some method within a subclass. Defaults to
a list with 'outline' and 'name'
"""
contents: Dict[str, List[str]] = dataclasses.field(default_factory=dict)
default: Any = dataclasses.field(default_factory=list)
components: sourdough.Library = sourdough.Library()
needs: ClassVar[Union[Sequence[str], str]] = ['outline', 'name']
""" Public Class Methods """
@classmethod
def from_outline(cls, outline: Outline, name: str) -> Workflow:
"""Creates a Workflow from an Outline.
Args:
outline (Outline): [description]
name (str): [description]
Returns:
Workflow: [description]
"""
workflow = cls()
workflow = cls._add_component(name=name,
outline=outline,
workflow=workflow)
for component in outline.components[name]:
workflow = cls._add_component(name=component,
outline=outline,
workflow=workflow)
return workflow
""" Public Methods """
def combine(self, workflow: Workflow) -> None:
"""Adds 'other' Workflow to this Workflow.
Combining creates an edge between every endpoint of this instance's
Workflow and the every root of 'workflow'.
Args:
workflow (Workflow): a second Workflow to combine with this one.
Raises:
ValueError: if 'workflow' has nodes that are also in 'flow'.
"""
if any(k in workflow.components.keys()
for k in self.components.keys()):
raise ValueError('Cannot combine Workflows with the same nodes')
else:
self.components.update(workflow.components)
super().combine(structure=workflow)
return self
def execute(self,
data: Any,
copy_components: bool = True,
**kwargs) -> Any:
"""Iterates over 'contents', using 'components'.
Args:
Returns:
"""
for path in iter(self):
data = self.execute_path(data=data,
path=path,
copy_components=copy_components,
**kwargs)
return data
def execute_path(self,
data: Any,
path: Sequence[str],
copy_components: bool = True,
**kwargs) -> Any:
"""Iterates over 'contents', using 'components'.
Args:
Returns:
"""
for node in more_itertools.always_iterable(path):
if copy_components:
component = copy.deepcopy(self.components[node])
else:
component = self.components[node]
data = component.execute(data=data, **kwargs)
return data
""" Private Class Methods """
@classmethod
def _add_component(cls, name: str, outline: Outline,
workflow: Workflow) -> Workflow:
"""[summary]
Args:
name (str): [description]
details (Details): [description]
workflow (Workflow): [description]
Returns:
Workflow: [description]
"""
workflow.append(node=name)
design = outline.designs[name]
component = cls.bases.component.library.borrow(names=[name, design])
instance = component.from_outline(name=name, outline=outline)
workflow.components[name] = instance
return workflow