def test_optional_output_when_stop_at_before_output_steps(
self, StrategyParser, FakeColumnSet, get_provider, read_config):
pynonymize(
input_path="TEST_INPUT",
strategyfile_path="TEST_STRATEGYFILE",
output_path=None,
db_type="TEST_TYPE",
db_host="TEST_HOST",
db_port="TEST_PORT",
db_name="TEST_NAME",
db_user="******",
db_password="******",
fake_locale="TEST_LOCALE",
start_at_step=None,
stop_at_step=None,
skip_steps=["DUMP_DB"],
)
StrategyParser.return_value.parse_config.assert_called()
get_provider.assert_called_with(
type="TEST_TYPE",
db_host="TEST_HOST",
db_port="TEST_PORT",
db_user="******",
db_pass="******",
db_name="TEST_NAME",
seed_rows=150,
)
provider = get_provider.return_value
provider.create_database.assert_called()
provider.restore_database.assert_called()
provider.anonymize_database.assert_called()
provider.dump_database.assert_not_called()
provider.drop_database.assert_called()
def test_pynonymize_only_step(self, StrategyParser, FakeColumnSet,
get_provider, read_config):
pynonymize(
input_path="TEST_INPUT",
strategyfile_path="TEST_STRATEGYFILE",
output_path="TEST_OUTPUT",
db_type="TEST_TYPE",
db_host="TEST_HOST",
db_port="TEST_PORT",
db_name="TEST_NAME",
db_user="******",
db_password="******",
fake_locale="TEST_LOCALE",
only_step="ANONYMIZE_DB",
)
StrategyParser.return_value.parse_config.assert_called()
get_provider.assert_called_with(
type="TEST_TYPE",
db_host="TEST_HOST",
db_port="TEST_PORT",
db_user="******",
db_pass="******",
db_name="TEST_NAME",
seed_rows=150,
)
provider = get_provider.return_value
provider.create_database.assert_not_called()
provider.restore_database.assert_not_called()
provider.anonymize_database.assert_called()
provider.dump_database.assert_not_called()
provider.drop_database.assert_not_called()
def test_any_db_kwarg(self, StrategyParser, FakeColumnSet, get_provider,
read_config):
"""
test that dynamic args are passed to the provider properly e.g. mssql_blah
"""
pynonymize(
input_path="TEST_INPUT",
strategyfile_path="TEST_STRATEGYFILE",
output_path="TEST_OUTPUT",
db_type="mssql",
db_host="TEST_HOST",
db_port="TEST_PORT",
db_name="TEST_NAME",
db_user="******",
db_password="******",
fake_locale="TEST_LOCALE",
mysql_other_amazing_var=
"TEST_DYNAMIC_VAR", # as this is mssql, this should be ignored
mssql_special_provider_var="TEST_DYNAMIC_VAR2",
)
StrategyParser.return_value.parse_config.assert_called()
get_provider.assert_called_with(
type="mssql",
db_host="TEST_HOST",
db_port="TEST_PORT",
db_user="******",
db_pass="******",
db_name="TEST_NAME",
seed_rows=150,
special_provider_var="TEST_DYNAMIC_VAR2",
)
def test_pynonymize_main_process(self, StrategyParser, FakeColumnSet, get_provider, read_config):
"""
a rough smoke test for the cli process. This needs an integration test to back it up.
"""
pynonymize(
input_path="TEST_INPUT",
strategyfile_path="TEST_STRATEGYFILE",
output_path="TEST_OUTPUT",
db_type="TEST_TYPE",
db_host="TEST_HOST",
db_port="TEST_PORT",
db_name="TEST_NAME",
db_user="******",
db_password="******",
fake_locale="TEST_LOCALE",
seed_rows=999
)
StrategyParser.return_value.parse_config.assert_called()
get_provider.assert_called_with(type="TEST_TYPE", db_host="TEST_HOST", db_port="TEST_PORT", db_user="******", db_pass="******", db_name="TEST_NAME", seed_rows=999)
provider = get_provider.return_value
provider.create_database.assert_called()
provider.restore_database.assert_called()
provider.anonymize_database.assert_called()
provider.dump_database.assert_called()
provider.drop_database.assert_called()
def test_pynonymize_missing_db_credentials():
with pytest.raises(ArgumentValidationError):
pynonymize(input_path="input.sql",
strategyfile_path="strategyfile.yml",
output_path="output.sql",
db_user=None,
db_password=None)
def test_pynonymize__when_ignore_errors_when_anonymize_raises__should_not_error(
self, StrategyParser, FakeColumnSet, get_provider, read_config):
get_provider.return_value.anonymize_database.side_effect = Exception(
"a problem!")
pynonymize(
input_path="TEST_INPUT",
strategyfile_path="TEST_STRATEGYFILE",
output_path="TEST_OUTPUT",
db_type="TEST_TYPE",
db_host="TEST_HOST",
db_port="TEST_PORT",
db_name="TEST_NAME",
db_user="******",
db_password="******",
fake_locale="TEST_LOCALE",
seed_rows=999,
ignore_anonymization_errors=True,
)
#!/usr/bin/env python3
from dotenv import load_dotenv
from pynonymizer.pynonymize import pynonymize
import os
load_dotenv()
pynonymize(input_path=os.getenv("DUMP_INPUT_FILE"),
strategyfile_path=os.getenv("STRATEGY_FILE"),
output_path=os.getenv("DUMP_OUTPUT_FILE"),
db_user=os.getenv("DB_USER"),
db_password=os.getenv("DB_PASSWORD"),
db_type=os.getenv("DB_TYPE"),
db_host=os.getenv("DB_HOST"),
db_name=os.getenv("DB_NAME"),
db_port=os.getenv("DB_PORT"),
fake_locale=os.getenv("LOCALE"),
verbose="true",
start_at_step="RESTORE_DB")
def cli(rawArgs=None):
"""
Main entry point for the command line. Parse the cmdargs, load env and call to the main process
:param args:
:return:
"""
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger.handlers.clear()
console_handler = logging.StreamHandler(sys.stderr)
console_handler.setLevel(logging.INFO)
console_handler.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(console_handler)
# find the dotenv from the current working dir rather than the execution location
dotenv_file = dotenv.find_dotenv(usecwd=True)
dotenv.load_dotenv(dotenv_path=dotenv_file)
parser = create_parser()
args = parser.parse_args(rawArgs)
# legacy positionals take precedence if specified
# This is to support those not using the new options/env fallbacks
input = args.legacy_input or args.input
strategyfile = args.legacy_strategyfile or args.strategyfile
output = args.legacy_output or args.output
if args.legacy_input:
logger.warning(
"Positional INPUT is deprecated. Use the -i/--input option instead."
)
if args.legacy_strategyfile:
logger.warning(
"Positional STRATEGYFILE is deprecated. Use the -s/--strategy option instead."
)
if args.legacy_output:
logger.warning(
"Positional OUTPUT is deprecated. Use the -o/--output option instead."
)
_warn_deprecated_env("DB_TYPE", "PYNONYMIZER_DB_TYPE")
_warn_deprecated_env("DB_HOST", "PYNONYMIZER_DB_HOST")
_warn_deprecated_env("DB_NAME", "PYNONYMIZER_DB_NAME")
_warn_deprecated_env("DB_USER", "PYNONYMIZER_DB_USER")
_warn_deprecated_env("DB_PASS", "PYNONYMIZER_DB_PASSWORD")
_warn_deprecated_env("FAKE_LOCALE", "PYNONYMIZER_FAKE_LOCALE")
if args.verbose:
console_handler.setLevel(logging.DEBUG)
try:
pynonymize(input_path=input,
strategyfile_path=strategyfile,
output_path=output,
db_type=args.db_type,
db_host=args.db_host,
db_port=args.db_port,
db_name=args.db_name,
db_user=args.db_user,
db_password=args.db_password,
fake_locale=args.fake_locale,
start_at_step=args.start_at_step,
skip_steps=args.skip_steps,
stop_at_step=args.stop_at_step,
seed_rows=args.seed_rows,
mssql_driver=args.mssql_driver,
mssql_backup_compression=args.mssql_backup_compression,
mysql_cmd_opts=args.mysql_cmd_opts,
mysql_dump_opts=args.mysql_dump_opts,
postgres_cmd_opts=args.postgres_cmd_opts,
postgres_dump_opts=args.postgres_dump_opts,
dry_run=args.dry_run,
verbose=args.verbose)
except ModuleNotFoundError as error:
if error.name == "pyodbc" and args.db_type == "mssql":
logger.error("Missing Required Packages for database support.")
logger.error(
"Install package extras: pip install pynonymizer[mssql]")
sys.exit(1)
else:
raise error
except ImportError as error:
if error.name == "pyodbc" and args.db_type == "mssql":
logger.error(
"Error importing pyodbc (mssql). "
"The ODBC driver may not be installed on your system. See package `unixodbc`."
)
sys.exit(1)
else:
raise error
except DatabaseConnectionError as error:
logger.error("Failed to connect to database.")
if args.verbose:
logger.error(error)
sys.exit(1)
except ArgumentValidationError as error:
logger.error(
"Missing values for required arguments: \n" +
"\n".join(error.validation_messages) +
"\nSet these using the command-line options or with environment variables. \n"
"For a complete list, See the program help below.\n")
parser.print_help()
sys.exit(2)
def main(rawArgs=None):
"""
Main entry point for the command line. Parse the cmdargs, load env and call to the main process
:param args:
:return:
"""
# find the dotenv from the current working dir rather than the execution location
dotenv_file = dotenv.find_dotenv(usecwd=True)
dotenv.load_dotenv(dotenv_path=dotenv_file)
parser = create_parser()
args = parser.parse_args(rawArgs)
# legacy positionals take precedence if specified
# This is to support those not using the new options/env fallbacks
input = args.legacy_input or args.input
strategyfile = args.legacy_strategyfile or args.strategyfile
output = args.legacy_output or args.output
try:
pynonymize(input_path=input,
strategyfile_path=strategyfile,
output_path=output,
db_type=args.db_type,
db_host=args.db_host,
db_port=args.db_port,
db_name=args.db_name,
db_user=args.db_user,
db_password=args.db_password,
fake_locale=args.fake_locale,
start_at_step=args.start_at_step,
skip_steps=args.skip_steps,
stop_at_step=args.stop_at_step,
seed_rows=args.seed_rows,
mssql_backup_compression=args.mssql_backup_compression,
mysql_dump_opts=args.mysql_dump_opts,
dry_run=args.dry_run,
verbose=args.verbose)
except ModuleNotFoundError as error:
if error.name == "pyodbc" and args.db_type == "mssql":
logger.error("Missing Required Packages for database support.")
logger.error(
"Install package extras: pip install pynonymizer[mssql]")
sys.exit(1)
else:
raise error
except ImportError as error:
if error.name == "pyodbc" and args.db_type == "mssql":
logger.error(
"Error importing pyodbc (mssql). "
"The ODBC driver may not be installed on your system. See package `unixodbc`."
)
sys.exit(1)
else:
raise error
except DatabaseConnectionError as error:
logger.error("Failed to connect to database.")
if args.verbose:
logger.error(error)
sys.exit(1)
except ArgumentValidationError as error:
logger.error(
"Missing values for required arguments: \n" +
"\n".join(error.validation_messages) +
"\nSet these using the command-line options or with environment variables. \n"
"For a complete list, See the program help below.\n")
parser.print_help()
sys.exit(2)
