def _create_storage(wok_conf):
storage_conf = wok_conf.get("storage")
if storage_conf is None:
storage_conf = wok_conf.create_element()
storage_type = storage_conf.get("type", "sfs")
if "work_path" not in storage_conf:
wok_work_path = wok_conf.get("work_path", os.path.join(os.getcwd(), "wok"))
storage_conf["work_path"] = os.path.join(wok_work_path, "storage")
return create_storage(storage_type, StorageContext.CONTROLLER, storage_conf)
def __init__(self):
# Get task key and storage configuration
cmd_conf = OptionsConfig(required=["instance_name", "module_path", "task_index", "storage.type"])
instance_name = cmd_conf["instance_name"]
module_path = cmd_conf["module_path"]
task_index = cmd_conf["task_index"]
storage_conf = cmd_conf["storage"]
storage = create_storage(storage_conf["type"], StorageContext.EXECUTION, storage_conf)
# Read data and configuration
self.data = storage.load_task_config(instance_name, module_path, task_index)
self.conf = self.data["conf"]
del self.data["conf"]
# set per task configuration values and expand
self.conf["__task_index"] = task_index
self.conf.expand_vars()
self.id = self.data["id"]
self.name = self.data["name"]
self.module_path = self.data["module"]
self.instance_name = self.data["instance"]
self._main = None
self._generators = []
self._foreach = None
self._begin = None
self._end = None
self._start_time = 0
self._end_time = self._start_time
logger.initialize(self.conf.get("log"))
self._log = self.logger()
# Initialize data iteration
iter_conf = self.data["iteration"]
self._iter_strategy = iter_conf.get("strategy")
self._iter_size = iter_conf.get("size", 0, dtype=int)
# Initialize ports
self._port_map = {}
self._in_ports = []
self._out_ports = []
if "ports" in self.data:
ports_conf = self.data["ports"]
for port_conf in ports_conf.get("in", []):
port = create_port(PORT_MODE_IN, port_conf, storage)
self._port_map[port.name] = port
self._in_ports += [port]
for port_conf in ports_conf.get("out", []):
port = create_port(PORT_MODE_OUT, port_conf, storage)
self._port_map[port.name] = port
self._out_ports += [port]
self.__ports_accessor = PortsAccessor(self._port_map)
# The context field is free to be used by the task user to
# save variables related with the whole task life cycle.
# By default it is initialized with a dictionary but can be
# overwrited with any value by the user. Wok will never use it.
self.context = {}
