def create_with_emr_cluster_id(config_path, cluster_mode,
destination_database,
destination_environment, algorithm_instance,
emr_cluster_id):
"""
Create algorithm configuration object from acon file. Method will discover acon file based on the
parameters passed to it.
:return: Returns algorithm configuration object of the type that is used for calling the method.
"""
# Create config service to get acon file path.
config_service = ConfigService(config_path)
acon_path = config_service.get_acon_path(cluster_mode,
destination_database,
destination_environment,
algorithm_instance)
acon_dict = Util.load_dict(acon_path)
environment = acon_dict[
AlgorithmConfigurationHadoop.Sections.ENVIRONMENT]
environment[
AlgorithmConfigurationHadoop.Keys.EMR_CLUSTER_ID] = emr_cluster_id
return AlgorithmConfigurationHadoop(algorithm_instance, acon_dict)
def read_acon_params(execution_system, table_name):
config_service = ConfigService(execution_system.config)
acon_path = config_service.get_acon_path(execution_system.database,
execution_system.environment,
table_name)
acon_dict = Util.load_dict(acon_path)
return acon_dict.get(LoadHadoop.PARAMETERS_KEY, {})
def setup_acon_from_file(config_dir_path, destination_database,
destination_environment, algorithm_instance,
base_acon_path):
acon_file_path = AconHelper.get_acon_file_path(
config_dir_path, destination_database, destination_environment,
algorithm_instance)
if not os.path.isdir(os.path.dirname(acon_file_path)):
os.makedirs(os.path.dirname(acon_file_path))
py.path.local(acon_file_path).write(
py.path.local(base_acon_path).read())
acon_dict = Util.load_dict(base_acon_path)
return acon_file_path, acon_dict
def create_with_ext_params(config_path, cluster_mode, destination_database,
destination_environment, algorithm_instance,
ext_params_str):
"""
Create algorithm configuration object from acon file. Method will discover acon file based on the
parameters passed to it.
:return: Returns algorithm configuration object of the type that is used for calling the method.
"""
# Create config service to get acon file path.
config_service = ConfigService(config_path)
acon_path = config_service.get_acon_path(cluster_mode,
destination_database,
destination_environment,
algorithm_instance)
acon_dict = Util.load_dict(acon_path)
if ext_params_str:
ext_params_dict = json.loads(ext_params_str)
acon_dict = Util.merge_nested_dicts(acon_dict, ext_params_dict)
return AlgorithmConfigurationHadoop(algorithm_instance, acon_dict)
def setup_oracle_scon(config_dir,
source_system,
db_cd,
base_scon_path,
database_type=None):
# Making sure that we can accept both strings as well as py.path.local objects.
config_dir = py.path.local(str(config_dir))
config_system_dir = config_dir.join("system")
config_credentials_dir = config_dir.join("credentials")
if not config_system_dir.check():
config_system_dir.mkdir()
if not config_credentials_dir.check():
config_credentials_dir.mkdir()
oracle_docker_ip = os.getenv("ORACLE_DOCKER_IP", "")
credentials_data = {
"oracle_conn_string": {
"lake":
"LAKE/test_lake_password@%s:1521/XE" % oracle_docker_ip,
"lake_out":
"LAKE_OUT/test_lake_out_password@%s:1521/XE" %
oracle_docker_ip,
"m3d":
"M3D/test_m3d_password@%s:1521/XE" % oracle_docker_ip,
"mart_mod":
"MART_MOD/test_mart_mod_password@%s:1521/XE" %
oracle_docker_ip,
"mart_cal":
"MART_CAL/test_mart_cal_password@%s:1521/XE" %
oracle_docker_ip,
"mart_out":
"MART_OUT/test_mart_out_password@%s:1521/XE" %
oracle_docker_ip,
"test_lake":
"TEST_LAKE/test_lake_password@%s:1521/XE" % oracle_docker_ip,
"test_lake_out":
"TEST_LAKE_OUT/test_lake_out_password@%s:1521/XE" %
oracle_docker_ip,
"test_mart_mod":
"TEST_MART_MOD/test_mart_mod_password@%s:1521/XE" %
oracle_docker_ip,
"test_mart_cal":
"TEST_MART_CAL/test_mart_cal_password@%s:1521/XE" %
oracle_docker_ip,
"test_mart_out":
"TEST_MART_OUT/test_mart_out_password@%s:1521/XE" %
oracle_docker_ip,
"dev_mart_mod":
"DEV_MART_MOD/dev_mart_mod_password@%s:1521/XE" %
oracle_docker_ip,
"dev_mart_cal":
"DEV_MART_CAL/dev_mart_cal_password@%s:1521/XE" %
oracle_docker_ip,
"dve_mart_out":
"DEV_MART_OUT/dev_mart_out_password@%s:1521/XE" %
oracle_docker_ip
}
}
credentials_filename = "credentials-{}-{}.json".format(
source_system, db_cd)
credentials_file = config_credentials_dir.join(credentials_filename)
credentials_file.write(json.dumps(credentials_data, indent=4))
scon_dict = Util.load_dict(base_scon_path)
scon_dict["credentials"] = str(credentials_file)
if database_type:
scon_dict["database_type"] = database_type
scon_filename = "scon-{}-{}.json".format(source_system, db_cd)
scon_file = config_system_dir.join(scon_filename)
scon_file.write(json.dumps(scon_dict, indent=4))
return str(scon_file), scon_dict