def _init_options(self, kwargs):
super(LoadData, self)._init_options(kwargs)
self.options["ignore_row_errors"] = process_bool_arg(
self.options.get("ignore_row_errors", False))
if self.options.get("database_url"):
# prefer database_url if it's set
self.options["sql_path"] = None
elif self.options.get("sql_path"):
self.options["sql_path"] = os_friendly_path(
self.options["sql_path"])
self.options["database_url"] = None
else:
raise TaskOptionsError(
"You must set either the database_url or sql_path option.")
self.reset_oids = self.options.get("reset_oids", True)
self.bulk_mode = (self.options.get("bulk_mode")
and self.options.get("bulk_mode").title())
if self.bulk_mode and self.bulk_mode not in ["Serial", "Parallel"]:
raise TaskOptionsError(
"bulk_mode must be either Serial or Parallel")
self.options["inject_namespaces"] = process_bool_arg(
self.options.get("inject_namespaces", True))
self.options["drop_missing_schema"] = process_bool_arg(
self.options.get("drop_missing_schema", False))
def _run_task(self):
self.mapping = self.get_combined_mapping()
with open(os_friendly_path(self.options.get("output_path")),
"w+") as f:
f.write("\n".join(
Util.print_mapping_as_list(
self.mapping, self.options.get("mapping_tab_size"))))
def _init_options(self, kwargs):
super()._init_options(kwargs)
self.options["settings_path"] = os_friendly_path(
self.options.get("settings_path"))
if self.options["settings_path"] is None or not os.path.isfile(
self.options["settings_path"]):
raise TaskOptionsError(
f"File {self.options['settings_path']} does not exist")
def _init_options(self, kwargs):
super(QueryData, self)._init_options(kwargs)
if self.options.get("sql_path"):
if self.options.get("database_url"):
raise TaskOptionsError(
"The database_url option is set dynamically with the sql_path option. Please unset the database_url option."
)
self.logger.info("Using in-memory sqlite database")
self.options["database_url"] = "sqlite://"
self.options["sql_path"] = os_friendly_path(self.options["sql_path"])
def _init_options(self, kwargs):
super(ExtractData, self)._init_options(kwargs)
if self.options.get("sql_path"):
if self.options.get("database_url"):
raise TaskOptionsError(
"The database_url option is set dynamically with the sql_path option. Please unset the database_url option."
)
self.logger.info("Using in-memory sqlite database")
self.options["database_url"] = "sqlite://"
self.options["sql_path"] = os_friendly_path(self.options["sql_path"])
def _init_options(self, kwargs):
super(ExtractData, self)._init_options(kwargs)
if self.options.get("database_url"):
# prefer database_url if it's set
self.options["sql_path"] = None
elif self.options.get("sql_path"):
self.logger.info("Using in-memory sqlite database")
self.options["database_url"] = "sqlite://"
self.options["sql_path"] = os_friendly_path(
self.options["sql_path"])
else:
raise TaskOptionsError(
"You must set either the database_url or sql_path option.")
def _init_options(self, kwargs):
super(LoadData, self)._init_options(kwargs)
if self.options.get("sql_path"):
if self.options.get("database_url"):
raise TaskOptionsError(
"The database_url option is set dynamically with the sql_path option. Please unset the database_url option."
)
self.options["sql_path"] = os_friendly_path(self.options["sql_path"])
if not os.path.isfile(self.options["sql_path"]):
raise TaskOptionsError(
"File {} does not exist".format(self.options["sql_path"])
)
self.logger.info("Using in-memory sqlite database")
self.options["database_url"] = "sqlite://"
def _init_options(self, kwargs):
super(LoadData, self)._init_options(kwargs)
if self.options.get("sql_path"):
if self.options.get("database_url"):
raise TaskOptionsError(
"The database_url option is set dynamically with the sql_path option. Please unset the database_url option."
)
self.options["sql_path"] = os_friendly_path(
self.options["sql_path"])
if not os.path.isfile(self.options["sql_path"]):
raise TaskOptionsError("File {} does not exist".format(
self.options["sql_path"]))
self.logger.info("Using in-memory sqlite database")
self.options["database_url"] = "sqlite://"
def _init_options(self, kwargs):
super(LoadData, self)._init_options(kwargs)
self.options["ignore_row_errors"] = process_bool_arg(
self.options.get("ignore_row_errors", False))
if self.options.get("database_url"):
# prefer database_url if it's set
self.options["sql_path"] = None
elif self.options.get("sql_path"):
self.options["sql_path"] = os_friendly_path(
self.options["sql_path"])
self.logger.info("Using in-memory sqlite database")
self.options["database_url"] = "sqlite://"
else:
raise TaskOptionsError(
"You must set either the database_url or sql_path option.")
def _init_options(self, kwargs):
super(LoadData, self)._init_options(kwargs)
self.options["ignore_row_errors"] = process_bool_arg(
self.options.get("ignore_row_errors", False))
if self.options.get("database_url"):
# prefer database_url if it's set
self.options["sql_path"] = None
elif self.options.get("sql_path"):
self.options["sql_path"] = os_friendly_path(
self.options["sql_path"])
self.options["database_url"] = None
else:
raise TaskOptionsError(
"You must set either the database_url or sql_path option.")
self.reset_oids = self.options.get("reset_oids", True)
def _init_options(self, kwargs):
super(ExtractData, self)._init_options(kwargs)
if self.options.get("database_url"):
# prefer database_url if it's set
self.options["sql_path"] = None
elif self.options.get("sql_path"):
self.options["sql_path"] = os_friendly_path(
self.options["sql_path"])
else:
raise TaskOptionsError(
"You must set either the database_url or sql_path option.")
inject_namespaces = self.options.get("inject_namespaces")
self.options["inject_namespaces"] = process_bool_arg(
True if inject_namespaces is None else inject_namespaces)
self.options["drop_missing_schema"] = process_bool_arg(
self.options.get("drop_missing_schema") or False)
def _init_options(self, kwargs):
super(LoadData, self)._init_options(kwargs)
self.options["ignore_row_errors"] = process_bool_arg(
self.options.get("ignore_row_errors", False))
if self.options.get("sql_path"):
if self.options.get("database_url"):
raise TaskOptionsError(
"The database_url option is set dynamically with the sql_path option. Please unset the database_url option."
)
self.options["sql_path"] = os_friendly_path(
self.options["sql_path"])
if not os.path.isfile(self.options["sql_path"]):
raise TaskOptionsError(
f"File {self.options['sql_path']} does not exist")
self.logger.info("Using in-memory sqlite database")
self.options["database_url"] = "sqlite://"
def _init_options(self, kwargs):
super(ExtractData, self)._init_options(kwargs)
if self.options.get("database_url"):
# prefer database_url if it's set
self.options["sql_path"] = None
elif self.options.get("sql_path"):
self.logger.info("Using in-memory sqlite database")
self.options["database_url"] = "sqlite://"
self.options["sql_path"] = os_friendly_path(
self.options["sql_path"])
else:
raise TaskOptionsError(
"You must set either the database_url or sql_path option.")
self.options["inject_namespaces"] = process_bool_arg(
self.options.get("inject_namespaces", True))
self.options["drop_missing_schema"] = process_bool_arg(
self.options.get("drop_missing_schema", False))
def get_available_package_mappings(self):
rows = [["PATH", "NAMESPACE"]]
available_mappings = {}
for directory in self.options["package_mapping_directories"]:
root = os_friendly_path(directory)
items = os.listdir(root)
items.sort()
is_first = True
for item in items:
path = os.path.join(root, item)
if os.path.isfile(os.path.join(
root, item)) and item.endswith(".yml"):
namespace = item[:-4]
mapping = {"path": path, "namespace": namespace}
available_mappings[namespace] = mapping
rows.append([root if is_first else "", namespace])
is_first = False
self.log_table(rows, groupByBlankColumns=True)
return available_mappings
def test_os_friendly_path(self):
with mock.patch("os.sep", "\\"):
self.assertEqual("\\", utils.os_friendly_path("/"))
def test_os_friendly_path(self):
with mock.patch("os.sep", "\\"):
assert "\\" == utils.os_friendly_path("/")
def test_os_friendly_path(self):
with mock.patch("os.sep", "\\"):
self.assertEqual("\\", utils.os_friendly_path("/"))
def get_mapping_from_file(mapping_path):
with open(os_friendly_path(mapping_path), "r") as f:
return yaml.safe_load(f)
