def __init__(self, log_name):
self.log_name = log_name
load_package_config('/nail/srv/configs/data_pipeline_tools.yaml')
self.config = get_config()
self.log = logging.getLogger(self.log_name)
self._setup_logging()
self.schematizer = get_schematizer()
def __init__(self, log_name):
self.log_name = log_name
load_package_config('/nail/srv/configs/data_pipeline_tools.yaml')
self.config = get_config()
self.log = logging.getLogger(self.log_name)
self._setup_logging()
self.schematizer = get_schematizer()
def process_commandline_options(self, args=None):
super(CompactionSetter, self).process_commandline_options(args=args)
load_package_config(self.options.config_path)
self.dry_run = self.options.dry_run
self.whitelist_topic = self.options.whitelist_topic
self.schematizer = get_schematizer()
def process_commandline_options(self, args=None):
super(FullRefreshRequester,
self).process_commandline_options(args=args)
if (self.options.avg_rows_per_second_cap is not None
and self.options.avg_rows_per_second_cap <= 0):
raise ValueError(
"--avg-rows-per-second-cap must be greater than 0")
if self.options.batch_size <= 0:
raise ValueError("--batch-size option must be greater than 0.")
if not self.options.source_id and not (self.options.source_name
and self.options.namespace):
raise ValueError(
"--source-id or both of--source-name and --namespace must be defined"
)
if self.options.source_id and (self.options.source_name
or self.options.namespace):
raise ValueError(
"Cannot use both --source-id and either of --namespace and --source-name"
)
load_package_config(self.options.config_path)
self.schematizer = get_schematizer()
source_ids = self.get_source_ids()
if len(source_ids) == 0:
raise ValueError(
"Found no sources with namespace_name {} and source_name {}".
format(self.options.namespace, self.options.source_name))
elif len(source_ids) > 1:
raise ValueError(
"Pair of namespace_name {} and source_name {} somehow received more than one"
" source. Investigation as to how is recommended.".format(
self.options.namespace, self.options.source_name))
self.source_id = source_ids[0]
def _init_global_state(self):
if self.options.namespace is None:
raise ValueError("--namespace is required to be defined")
self.namespace = self.options.namespace
self._set_cluster_and_database()
self.config_path = self.options.config_path
self.dry_run = self.options.dry_run
load_package_config(self.config_path)
# Removing the cmd line arguments to prevent child process error.
sys.argv = sys.argv[:1]
def _init_global_state(self):
if self.options.namespace is None:
raise ValueError("--namespace is required to be defined")
self.namespace = self.options.namespace
self._set_cluster_and_database()
self.config_path = self.options.config_path
self.dry_run = self.options.dry_run
load_package_config(self.config_path)
# Removing the cmd line arguments to prevent child process error.
sys.argv = sys.argv[:1]
def _init_global_state(self):
if self.options.database is None:
raise ValueError("--database is required to be defined")
self.cluster = self.options.cluster
self.database = self.options.database
self.namespace = DBSourcedNamespace(cluster=self.cluster,
database=self.database).get_name()
self.config_path = self.options.config_path
self.dry_run = self.options.dry_run
self.per_source_throughput_cap = self.options.per_source_throughput_cap
self.total_throughput_cap = self.options.total_throughput_cap
load_package_config(self.config_path)
self.refresh_runner_path = self.get_refresh_runner_path()
# Removing the cmd line arguments to prevent child process error.
sys.argv = sys.argv[:1]
def test_load_package_config(self, mock_staticconf, mock_config_packages):
filename = '/path/to/a/file'
config = config_util.load_package_config(filename)
mock_staticconf.YamlConfiguration.assert_called_with(filename)
file_contents = mock_staticconf.YamlConfiguration.return_value
file_contents.get.assert_called_with('module_config')
mock_config_packages.assert_called_with(
file_contents.get.return_value,
flatten=True,
)
assert file_contents == config
def test_load_package_config(self, mock_staticconf, mock_config_packages):
filename = '/path/to/a/file'
config = config_util.load_package_config(filename)
mock_staticconf.YamlConfiguration.assert_called_with(filename)
file_contents = mock_staticconf.YamlConfiguration.return_value
file_contents.get.assert_called_with('module_config')
mock_config_packages.assert_called_with(
file_contents.get.return_value,
flatten=True,
)
assert file_contents == config
def process_commandline_options(self, args=None):
super(FullRefreshJob, self).process_commandline_options(args=args)
if (self.options.avg_rows_per_second_cap is not None and
self.options.avg_rows_per_second_cap <= 0):
raise ValueError("--avg-rows-per-second-cap must be greater than 0")
if self.options.batch_size <= 0:
raise ValueError("--batch-size option must be greater than 0.")
if not self.options.source_id and not (
self.options.source_name and
self.options.namespace
):
raise ValueError("--source-id or both of--source-name and --namespace must be defined")
if self.options.source_id and (
self.options.source_name or
self.options.namespace
):
raise ValueError("Cannot use both --source-id and either of --namespace and --source-name")
load_package_config(self.options.config_path)
self.schematizer = get_schematizer()
def process_commandline_options(self, args=None):
super(FullRefreshJob, self).process_commandline_options(args=args)
if (self.options.avg_rows_per_second_cap is not None
and self.options.avg_rows_per_second_cap <= 0):
raise ValueError(
"--avg-rows-per-second-cap must be greater than 0")
if self.options.batch_size <= 0:
raise ValueError("--batch-size option must be greater than 0.")
if not self.options.source_id and not (self.options.source_name
and self.options.namespace):
raise ValueError(
"--source-id or both of--source-name and --namespace must be defined"
)
if self.options.source_id and (self.options.source_name
or self.options.namespace):
raise ValueError(
"Cannot use both --source-id and either of --namespace and --source-name"
)
load_package_config(self.options.config_path)
self.schematizer = get_schematizer()