def get_cluster_from_cluster_dict(cluster_dict, kms_region):
cluster = RedshiftCluster(cluster_dict['clusterEndpoint'])
cluster.set_port(cluster_dict['clusterPort'])
cluster.set_user(cluster_dict['connectUser'])
cluster.set_host(cluster_dict['clusterEndpoint'])
cluster.set_db(cluster_dict['db'])
if 'connectPwd' in cluster_dict:
if kms_region is None:
kms_region = cluster.get_region_name()
kms_helper = KMSHelper(kms_region)
cluster.set_password(kms_helper.decrypt(cluster_dict['connectPwd']))
cluster.set_user_auto_create(False)
if 'userAutoCreate' in cluster_dict \
and cluster_dict['userAutoCreate'].lower() == 'true':
cluster.set_user_auto_create(True)
cluster.user_db_groups = []
if 'userDbGroups' in cluster_dict:
cluster.set_user_db_groups(cluster_dict['userDbGroups'])
return cluster
def __init__(self, config_helper, source_table, encryption_key_id=None):
if 's3Staging' not in config_helper.config:
raise S3Details.NoS3StagingInformationFoundException()
else:
s3_staging_conf = config_helper.config['s3Staging']
if 'region' in s3_staging_conf:
self.dataStagingRegion = s3_staging_conf['region']
else:
logging.warning('No region in s3_staging_conf')
self.dataStagingRegion = None
if 'deleteOnSuccess' in s3_staging_conf \
and s3_staging_conf['deleteOnSuccess'].lower() == 'true':
self.deleteOnSuccess = True
else:
self.deleteOnSuccess = False
if 'path' in s3_staging_conf:
# datetime alias for operations
self.nowString = "{:%Y-%m-%d_%H:%M:%S}".format(datetime.datetime.now())
self.dataStagingRoot = "{s3_stage_path}/{timestamp}-{table_name}/".format(
s3_stage_path=s3_staging_conf['path'].rstrip("/"),
timestamp=self.nowString,
table_name=source_table.get_table()
)
self.dataStagingPath = "{root}{db_name}.{schema_name}.{table_name}".format(
root=self.dataStagingRoot,
db_name=source_table.get_db(),
schema_name=source_table.get_schema(),
table_name=source_table.get_table())
if not self.dataStagingPath or not self.dataStagingPath.startswith("s3://"):
raise S3Details.S3StagingPathMustStartWithS3
if 'aws_iam_role' in s3_staging_conf:
role = s3_staging_conf['aws_iam_role']
self.access_credentials = S3AccessCredentialsRole(role)
elif 'aws_access_key_id' in s3_staging_conf and 'aws_secret_access_key' in s3_staging_conf:
kms_helper = KMSHelper(config_helper.s3_helper.region_name)
key_id = kms_helper.decrypt(s3_staging_conf['aws_access_key_id']).decode('utf-8')
secret_key = kms_helper.decrypt(s3_staging_conf['aws_secret_access_key']).decode('utf-8')
self.access_credentials = S3AccessCredentialsKey(key_id, secret_key)
else:
raise(S3Details.NoS3CredentialsFoundException())
use_kms = True
if 'kmsGeneratedKey' in s3_staging_conf:
if s3_staging_conf['kmsGeneratedKey'].lower() == 'false':
use_kms = False
if use_kms:
kms_helper = KMSHelper(config_helper.s3_helper.region_name)
self.symmetric_key = kms_helper.generate_base64_encoded_data_key(encryption_key_id)
else:
self.symmetric_key = base64.b64encode(KMSHelper.generate_data_key_without_kms())
# noinspection PyBroadException
try:
self.symmetric_key = self.symmetric_key.decode('utf-8')
except:
logging.debug('Exception converting string can be ignored, likely Python2 so already a string.')
def get_cluster_from_cluster_dict(cluster_dict, kms_region):
cluster = RedshiftCluster(cluster_dict['clusterEndpoint'])
cluster.set_port(cluster_dict['clusterPort'])
cluster.set_user(cluster_dict['connectUser'])
cluster.set_host(cluster_dict['clusterEndpoint'])
cluster.set_db(cluster_dict['db'])
if 'connectPwd' in cluster_dict:
if kms_region is None:
kms_region = cluster.get_region_name()
kms_helper = KMSHelper(kms_region)
cluster.set_password(kms_helper.decrypt(
cluster_dict['connectPwd']))
cluster.set_user_auto_create(False)
if 'userAutoCreate' in cluster_dict \
and cluster_dict['userAutoCreate'].lower() == 'true':
cluster.set_user_auto_create(True)
cluster.user_db_groups = []
if 'userDbGroups' in cluster_dict:
cluster.set_user_db_groups(cluster_dict['userDbGroups'])
return cluster
def __init__(self, config_helper, source_table, encryption_key_id=None):
if 's3Staging' not in config_helper.config:
raise S3Details.NoS3StagingInformationFoundException()
else:
s3_staging_conf = config_helper.config['s3Staging']
if 'region' in s3_staging_conf:
self.dataStagingRegion = s3_staging_conf['region']
else:
logging.warning('No region in s3_staging_conf')
self.dataStagingRegion = None
if 'deleteOnSuccess' in s3_staging_conf \
and s3_staging_conf['deleteOnSuccess'].lower() == 'true':
self.deleteOnSuccess = True
else:
self.deleteOnSuccess = False
if 'path' in s3_staging_conf:
# datetime alias for operations
self.nowString = "{:%Y-%m-%d_%H:%M:%S}".format(
datetime.datetime.now())
self.dataStagingRoot = "{s3_stage_path}/{timestamp}/".format(
s3_stage_path=s3_staging_conf['path'].rstrip("/"),
timestamp=self.nowString)
self.dataStagingPath = "{root}{db_name}.{schema_name}.{table_name}".format(
root=self.dataStagingRoot,
db_name=source_table.get_db(),
schema_name=source_table.get_schema(),
table_name=source_table.get_table())
if not self.dataStagingPath or not self.dataStagingPath.startswith(
"s3://"):
raise S3Details.S3StagingPathMustStartWithS3
if 'aws_iam_role' in s3_staging_conf:
role = s3_staging_conf['aws_iam_role']
self.access_credentials = S3AccessCredentialsRole(role)
elif 'aws_access_key_id' in s3_staging_conf and 'aws_secret_access_key' in s3_staging_conf:
kms_helper = KMSHelper(config_helper.s3_helper.region_name)
key_id = kms_helper.decrypt(
s3_staging_conf['aws_access_key_id'])
secret_key = kms_helper.decrypt(
s3_staging_conf['aws_secret_access_key'])
self.access_credentials = S3AccessCredentialsKey(
key_id, secret_key)
else:
raise (S3Details.NoS3CredentialsFoundException())
use_kms = True
if 'kmsGeneratedKey' in s3_staging_conf:
if s3_staging_conf['kmsGeneratedKey'].lower() == 'false':
use_kms = False
if use_kms:
kms_helper = KMSHelper(config_helper.s3_helper.region_name)
self.symmetric_key = kms_helper.generate_base64_encoded_data_key(
encryption_key_id)
else:
self.symmetric_key = base64.b64encode(
KMSHelper.generate_data_key_without_kms())
# noinspection PyBroadException
try:
self.symmetric_key = self.symmetric_key.decode('utf-8')
except:
logging.debug(
'Exception converting string can be ignored, likely Python2 so already a string.'
)