def load_table(config,
destination_system,
destination_database,
destination_environment,
destination_table,
load_type,
emr_cluster_id=None,
spark_params=None):
ds = DataSystem(config, destination_system, destination_database,
destination_environment)
# hadoop
if ds.database_type == DataSystem.DatabaseType.EMR:
if ds.storage_type == DataSystem.StorageType.S3:
from m3d.hadoop.load.load_executor_hadoop import LoadExecutorHadoop
LoadExecutorHadoop.create(
config_path=config,
destination_system=destination_system,
destination_database=destination_database,
destination_environment=destination_environment,
destination_table=destination_table,
load_type=load_type,
emr_cluster_id=emr_cluster_id,
spark_params_str=spark_params).run()
else:
raise m3d_exceptions.M3DUnsupportedStorageException(
ds.storage_type)
else:
raise m3d_exceptions.M3DUnsupportedDatabaseTypeException(
ds.database_type)
def drop_table(config,
destination_system,
destination_database,
destination_environment,
destination_table,
emr_cluster_id=None):
# create abstract table object to retrieve source technology
abstract_table = Table(config, destination_system,
destination_database, destination_environment,
destination_table)
destination_system_technology = abstract_table.get_destination_technology(
)
# hadoop
if destination_system_technology == DataSystem.SystemTechnology.HIVE:
if abstract_table.storage_type == DataSystem.StorageType.S3:
from m3d.hadoop.emr.emr_system import EMRSystem
emr_system = EMRSystem(config, destination_system,
destination_database,
destination_environment, emr_cluster_id)
emr_system.add_cluster_tag(EMRSystem.EMRClusterTag.API_METHOD,
M3D.drop_table.__name__)
emr_system.drop_table(destination_table)
else:
raise m3d_exceptions.M3DUnsupportedStorageException(
abstract_table.storage_type)
else:
raise m3d_exceptions.M3DUnsupportedDestinationSystemException(
destination_system_technology)
def load_table(config,
cluster_mode,
destination_system,
destination_database,
destination_environment,
destination_table,
load_type,
emr_cluster_id=None,
spark_params=None):
# create abstract table object to retrieve source technology
abstract_table = Table(config, cluster_mode, destination_system,
destination_database, destination_environment,
destination_table)
destination_system_technology = abstract_table.get_destination_technology(
)
# hadoop
if destination_system_technology == DataSystem.SystemTechnology.HIVE:
if abstract_table.storage_type == DataSystem.StorageType.S3:
from m3d.hadoop.load.load_executor_hadoop import LoadExecutorHadoop
LoadExecutorHadoop.create(
config_path=config,
cluster_mode=cluster_mode,
destination_system=destination_system,
destination_database=destination_database,
destination_environment=destination_environment,
destination_table=destination_table,
load_type=load_type,
emr_cluster_id=emr_cluster_id,
spark_params_str=spark_params).run()
else:
raise m3d_exceptions.M3DUnsupportedStorageException(
abstract_table.storage_type)
else:
raise m3d_exceptions.M3DUnsupportedDestinationSystemException(
destination_system_technology)