def __init__(self, name, source=None, destination=None):
# Initialize properties
self._source = None
self._destination = None
self._name = name
# Check to see if workflow yaml file exists. If so, set workflow configurations from file.
default_filepath = self._get_default_filepath(name)
backup_filepath = self._get_backup_filepath(name)
if os.path.exists(default_filepath):
log.info("Config file detected: {0}".format(default_filepath))
self._set_workflow_config(default_filepath)
elif os.path.exists(backup_filepath):
log.info("Config file detected: {0}".format(backup_filepath))
self._set_workflow_config(backup_filepath)
else:
log.info("No config file detected.")
# If source or destination are passed in as parameters, update source and dest configurations.
if source:
self._source = source
self._io_reader = Factory.get_reader(self._source["type"],
self._source)
if destination:
self._destination = destination
self._io_writer = Factory.get_writer(self._destination["type"],
self._destination)
def _set_workflow_config(self, yaml_file):
# Receives a yaml file path with Workflow configurations and sets appropriate values for properties in this class
log.info("Setting configurations from config file {0}".format(yaml_file))
try:
config = None
with open(yaml_file, "r") as ymlfile:
config = yaml.load(ymlfile)
self._source = config["source"]
self._destination = config["destination"]
self._io_reader = Factory.get_reader(self._source["type"], self._source)
self._io_writer = Factory.get_writer(
self._destination["type"], self._destination
)
# Set attributes for custom workflow properties
for key in config.keys():
if key not in self.DEFAULT_PARAMS:
setattr(self, key, config[key])
except Exception:
log.error(
"Error creating I/O reader and writer. Please check configurations in workflow config file at {0}".format(
yaml_file
)
)
raise
def set_destination(self, destination):
"""
Set destination.
:param destination: dict of configuration parameters for the destination (writer)
"""
self._destination = destination
self._io_writer = Factory.get_writer(self.source["destination"],
self.destination)
def test_get_io_writer_fs(fs_writer_config):
writer = Factory.get_writer("fs", fs_writer_config)
expected_cls = FileSystemWriter
assert isinstance(writer, expected_cls)
def test_get_io_writer_kafka(kafka_config):
writer = Factory.get_writer("kafka", kafka_config)
expected_cls = KafkaWriter
assert isinstance(writer, expected_cls)
def set_destination(self, destination):
self._destination = destination
self._io_writer = Factory.get_writer(self.source["destination"],
self.destination)