def _init_options(self, kwargs): super(LoadData, self)._init_options(kwargs) self.options["ignore_row_errors"] = process_bool_arg( self.options.get("ignore_row_errors", False)) if self.options.get("database_url"): # prefer database_url if it's set self.options["sql_path"] = None elif self.options.get("sql_path"): self.options["sql_path"] = os_friendly_path( self.options["sql_path"]) self.options["database_url"] = None else: raise TaskOptionsError( "You must set either the database_url or sql_path option.") self.reset_oids = self.options.get("reset_oids", True) self.bulk_mode = (self.options.get("bulk_mode") and self.options.get("bulk_mode").title()) if self.bulk_mode and self.bulk_mode not in ["Serial", "Parallel"]: raise TaskOptionsError( "bulk_mode must be either Serial or Parallel") self.options["inject_namespaces"] = process_bool_arg( self.options.get("inject_namespaces", True)) self.options["drop_missing_schema"] = process_bool_arg( self.options.get("drop_missing_schema", False))
def _run_task(self): self.mapping = self.get_combined_mapping() with open(os_friendly_path(self.options.get("output_path")), "w+") as f: f.write("\n".join( Util.print_mapping_as_list( self.mapping, self.options.get("mapping_tab_size"))))
def _init_options(self, kwargs): super()._init_options(kwargs) self.options["settings_path"] = os_friendly_path( self.options.get("settings_path")) if self.options["settings_path"] is None or not os.path.isfile( self.options["settings_path"]): raise TaskOptionsError( f"File {self.options['settings_path']} does not exist")
def _init_options(self, kwargs): super(QueryData, self)._init_options(kwargs) if self.options.get("sql_path"): if self.options.get("database_url"): raise TaskOptionsError( "The database_url option is set dynamically with the sql_path option. Please unset the database_url option." ) self.logger.info("Using in-memory sqlite database") self.options["database_url"] = "sqlite://" self.options["sql_path"] = os_friendly_path(self.options["sql_path"])
def _init_options(self, kwargs): super(ExtractData, self)._init_options(kwargs) if self.options.get("sql_path"): if self.options.get("database_url"): raise TaskOptionsError( "The database_url option is set dynamically with the sql_path option. Please unset the database_url option." ) self.logger.info("Using in-memory sqlite database") self.options["database_url"] = "sqlite://" self.options["sql_path"] = os_friendly_path(self.options["sql_path"])
def _init_options(self, kwargs): super(ExtractData, self)._init_options(kwargs) if self.options.get("database_url"): # prefer database_url if it's set self.options["sql_path"] = None elif self.options.get("sql_path"): self.logger.info("Using in-memory sqlite database") self.options["database_url"] = "sqlite://" self.options["sql_path"] = os_friendly_path( self.options["sql_path"]) else: raise TaskOptionsError( "You must set either the database_url or sql_path option.")
def _init_options(self, kwargs): super(LoadData, self)._init_options(kwargs) if self.options.get("sql_path"): if self.options.get("database_url"): raise TaskOptionsError( "The database_url option is set dynamically with the sql_path option. Please unset the database_url option." ) self.options["sql_path"] = os_friendly_path(self.options["sql_path"]) if not os.path.isfile(self.options["sql_path"]): raise TaskOptionsError( "File {} does not exist".format(self.options["sql_path"]) ) self.logger.info("Using in-memory sqlite database") self.options["database_url"] = "sqlite://"
def _init_options(self, kwargs): super(LoadData, self)._init_options(kwargs) if self.options.get("sql_path"): if self.options.get("database_url"): raise TaskOptionsError( "The database_url option is set dynamically with the sql_path option. Please unset the database_url option." ) self.options["sql_path"] = os_friendly_path( self.options["sql_path"]) if not os.path.isfile(self.options["sql_path"]): raise TaskOptionsError("File {} does not exist".format( self.options["sql_path"])) self.logger.info("Using in-memory sqlite database") self.options["database_url"] = "sqlite://"
def _init_options(self, kwargs): super(LoadData, self)._init_options(kwargs) self.options["ignore_row_errors"] = process_bool_arg( self.options.get("ignore_row_errors", False)) if self.options.get("database_url"): # prefer database_url if it's set self.options["sql_path"] = None elif self.options.get("sql_path"): self.options["sql_path"] = os_friendly_path( self.options["sql_path"]) self.logger.info("Using in-memory sqlite database") self.options["database_url"] = "sqlite://" else: raise TaskOptionsError( "You must set either the database_url or sql_path option.")
def _init_options(self, kwargs): super(LoadData, self)._init_options(kwargs) self.options["ignore_row_errors"] = process_bool_arg( self.options.get("ignore_row_errors", False)) if self.options.get("database_url"): # prefer database_url if it's set self.options["sql_path"] = None elif self.options.get("sql_path"): self.options["sql_path"] = os_friendly_path( self.options["sql_path"]) self.options["database_url"] = None else: raise TaskOptionsError( "You must set either the database_url or sql_path option.") self.reset_oids = self.options.get("reset_oids", True)
def _init_options(self, kwargs): super(ExtractData, self)._init_options(kwargs) if self.options.get("database_url"): # prefer database_url if it's set self.options["sql_path"] = None elif self.options.get("sql_path"): self.options["sql_path"] = os_friendly_path( self.options["sql_path"]) else: raise TaskOptionsError( "You must set either the database_url or sql_path option.") inject_namespaces = self.options.get("inject_namespaces") self.options["inject_namespaces"] = process_bool_arg( True if inject_namespaces is None else inject_namespaces) self.options["drop_missing_schema"] = process_bool_arg( self.options.get("drop_missing_schema") or False)
def _init_options(self, kwargs): super(LoadData, self)._init_options(kwargs) self.options["ignore_row_errors"] = process_bool_arg( self.options.get("ignore_row_errors", False)) if self.options.get("sql_path"): if self.options.get("database_url"): raise TaskOptionsError( "The database_url option is set dynamically with the sql_path option. Please unset the database_url option." ) self.options["sql_path"] = os_friendly_path( self.options["sql_path"]) if not os.path.isfile(self.options["sql_path"]): raise TaskOptionsError( f"File {self.options['sql_path']} does not exist") self.logger.info("Using in-memory sqlite database") self.options["database_url"] = "sqlite://"
def _init_options(self, kwargs): super(ExtractData, self)._init_options(kwargs) if self.options.get("database_url"): # prefer database_url if it's set self.options["sql_path"] = None elif self.options.get("sql_path"): self.logger.info("Using in-memory sqlite database") self.options["database_url"] = "sqlite://" self.options["sql_path"] = os_friendly_path( self.options["sql_path"]) else: raise TaskOptionsError( "You must set either the database_url or sql_path option.") self.options["inject_namespaces"] = process_bool_arg( self.options.get("inject_namespaces", True)) self.options["drop_missing_schema"] = process_bool_arg( self.options.get("drop_missing_schema", False))
def get_available_package_mappings(self): rows = [["PATH", "NAMESPACE"]] available_mappings = {} for directory in self.options["package_mapping_directories"]: root = os_friendly_path(directory) items = os.listdir(root) items.sort() is_first = True for item in items: path = os.path.join(root, item) if os.path.isfile(os.path.join( root, item)) and item.endswith(".yml"): namespace = item[:-4] mapping = {"path": path, "namespace": namespace} available_mappings[namespace] = mapping rows.append([root if is_first else "", namespace]) is_first = False self.log_table(rows, groupByBlankColumns=True) return available_mappings
def test_os_friendly_path(self): with mock.patch("os.sep", "\\"): self.assertEqual("\\", utils.os_friendly_path("/"))
def test_os_friendly_path(self): with mock.patch("os.sep", "\\"): assert "\\" == utils.os_friendly_path("/")
def get_mapping_from_file(mapping_path): with open(os_friendly_path(mapping_path), "r") as f: return yaml.safe_load(f)