def _dataload(self, subtask_options):
subtask_config = TaskConfig({"options": subtask_options})
subtask = LoadData(
project_config=self.project_config,
task_config=subtask_config,
org_config=self.org_config,
flow=self.flow,
name=self.name,
stepnum=self.stepnum,
)
subtask()
def _run_task(self):
mapping_file = os.path.abspath(self.options["mapping_yaml"])
debug_db_path = self.options.get("debug_db_path")
with temporary_dir() as tempdir:
if(debug_db_path):
sqlite_path = debug_db_path
else:
sqlite_path = os.path.join(tempdir, "generated_data.db")
url = "sqlite:///" + sqlite_path
self._generate_data(url, mapping_file)
subtask_config = TaskConfig(
{"options": {"database_url": url, "mapping": mapping_file}}
)
subtask = LoadData(
project_config=self.project_config,
task_config=subtask_config,
org_config=self.org_config,
flow=self.flow,
name=self.name,
stepnum=self.stepnum,
)
subtask()