def test_initsync_table(mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
mockargv_config = utils.merge_dicts(mockargv_config, {
"sourcedbtype": const.ORACLE,
"targetdbtype": const.POSTGRES,
})
mockargv = mocker.Mock(**mockargv_config)
source_conn_detail = dbuser.get_dbuser_properties(mockargv.sourceuser)
target_conn_detail = dbuser.get_dbuser_properties(mockargv.targetuser)
source_schema = "myschema"
target_schema = "myschema"
process_control_id = 1
query_condition = None
mock_msg_queue_config = {"put.return_value": None}
mock_msg_queue = mocker.Mock(**mock_msg_queue_config)
mockdb = build_mockdb(mocker, db_config, const.ORACLE)
db = build_initsyncdb(mocker, mockdb, mockargv, logger)
mock_update_ssp = mocker.patch("data_pipeline.initsync_pipe._update_source_system_profile")
# We don't want a real process being forked during tests
mock_process = mocker.patch("data_pipeline.initsync_pipe.Process")
initsync_pipe.initsync_table(mockargv, source_conn_detail,
target_conn_detail, source_schema, table.name, target_schema,
process_control_id, query_condition, mock_msg_queue)
assert mock_update_ssp.call_count == 2
def parallelise_initsync(argv, ssp_params, process_control_id, logger):
# Pivot the collection of source_system_profile records into
# three separate lists to enable us to call pool.map on each record
(source_schemas, tables, target_schemas,
query_conditions) = map(list, zip(*ssp_params))
source_conn_detail = dbuser.get_dbuser_properties(argv.sourceuser)
target_conn_detail = dbuser.get_dbuser_properties(argv.targetuser)
logger.info("Processing tables with {} dedicated worker processes".format(
argv.numprocesses))
pool = Pool(nodes=argv.numprocesses)
argvs = [argv] * len(tables)
source_conn_details = [source_conn_detail] * len(tables)
target_conn_details = [target_conn_detail] * len(tables)
pcids = [process_control_id] * len(tables)
queues = [manager.Queue()] * len(tables)
logger.debug("Starting a new process for each table in: {tables}".format(
tables=tables))
# Execute initsync for each schema/table combination in parallel
pool.map(initsync_table,
argvs,
source_conn_details,
target_conn_details,
source_schemas,
tables,
target_schemas,
pcids,
query_conditions,
queues,
chunksize=1) # Ensure tables are processed in sequence
# and workers are fully utilised
pool.close()
logger.debug("parallelise_initsync: Pool joining")
pool.join()
logger.debug("parallelise_initsync: Pool joined")
all_table_results = {}
for q in queues:
size = q.qsize()
message = q.get()
logger.debug("Message queue size = {s}, message = {m}".format(
s=size, m=message))
all_table_results.update(message)
logger.debug("all_table_results = {r}".format(r=all_table_results))
return all_table_results
def test_get_dbuser_properties_data_dir(
input_data_dir, input_datafile, expect_error, tmpdir):
data_dir = None
if input_data_dir:
data_dir = tmpdir.mkdir(input_data_dir)
if input_datafile and data_dir:
filename = str(data_dir.join(input_datafile))
open(filename, 'a').close()
data_dir = str(data_dir)
if expect_error:
with pytest.raises(ValueError) as e:
conn_details = get_dbuser_properties(data_dir)
assert str(e.value).startswith(expect_error)
else:
conn_details = get_dbuser_properties(data_dir)
assert conn_details.data_dir == data_dir
def __init__(self, mode, argv, audit_factory):
signal.signal(signal.SIGINT, self.exit_gracefully)
signal.signal(signal.SIGTERM, self.exit_gracefully)
self._mode = mode
self._argv = argv
self._logger = logging.getLogger(__name__)
details = dbuser.get_dbuser_properties(self._argv.audituser)
self._audit_conn_details = details
self._audit_factory = audit_factory
self._pc = audit_factory.build_process_control(self._mode)
def get_audit_db(argv):
audit_conn_details = dbuser.get_dbuser_properties(argv.audituser)
if audit_conn_details is None:
yield None
return
db = db_factory.build(const.POSTGRES)
db.connect(audit_conn_details)
try:
yield db
finally:
db.close()
def test_get_dbuser_properties(mocker, input_dbuser, input_password, keyring_password, expect_getpass_called_times, expect_userid, expect_password, expect_host, expect_port, expect_dbsid, expect_error):
get_password_func = mocker.patch("data_pipeline.utils.dbuser.keyring.get_password")
get_password_func.return_value = keyring_password
getpass_func = mocker.patch("data_pipeline.utils.dbuser.getpass.getpass")
getpass_func.return_value = input_password
set_password_func = mocker.patch("data_pipeline.utils.dbuser.keyring.set_password")
if expect_error:
with pytest.raises(ValueError) as e:
conn_details = get_dbuser_properties(input_dbuser)
print(conn_details)
assert str(e.value).startswith(expect_error)
else:
conn_details = get_dbuser_properties(input_dbuser)
assert conn_details.userid == expect_userid
assert conn_details.password == expect_password
assert conn_details.host == expect_host
assert conn_details.port == expect_port
assert conn_details.dbsid == expect_dbsid
assert getpass_func.call_count == expect_getpass_called_times
def __init__(self, table_name, argv):
self.id = 0
self.argv = argv
self._logger = logging.getLogger(__name__)
self._audit_conn_details = dbuser.get_dbuser_properties(
self.argv.audituser)
self._db = None
if self._audit_conn_details:
self._db = dbfactory.build(const.POSTGRES)
self._table = TableName(self.argv.auditschema, table_name)
self._fieldnames = []
self._additional_update_fields = []
def test_get_dbuser_properties_empty():
conn_details = get_dbuser_properties('')
assert conn_details is None
def connect_to_source_db(self):
self._logger.info("Extractor connecting to db")
conn_details = dbuser.get_dbuser_properties(self._argv.sourceuser)
self._source_db.connect(conn_details)
def __init__(self, argv):
self._argv = argv
self._audit_conn_details = dbuser.get_dbuser_properties(
self._argv.audituser)
self.session = audit_conn_factory.build_session(
self._audit_conn_details)