def event_handler(event, context):
try:
currentRegion = os.environ[region_key]
if currentRegion == None or currentRegion == '':
raise KeyError
except KeyError:
raise Exception("Unable to resolve environment variable %s" %
region_key)
kmsConnection = boto3.client('kms', region_name=currentRegion)
# KMS crypto authorisation context
authContext = utils.get_encryption_context(currentRegion)
global config
# lazy load the configuration file
if config == None:
try:
config_file = open("config.json", 'r')
config = json.load(config_file)
if config == None:
raise Exception("No Configuration Found")
except:
print sys.exc_info()[0]
raise
# resolve password
configDetail = config["configuration"]
encryptedPassword = configDetail["dbPassword"]
encryptedPassword = base64.b64decode(encryptedPassword)
if encryptedPassword != "" and encryptedPassword != None:
# decrypt the password using KMS
usePassword = kmsConnection.decrypt(
CiphertextBlob=encryptedPassword,
EncryptionContext=authContext)['Plaintext']
else:
raise Exception(
"Unable to run Utilities without a configured Password")
# run the column encoding utility, if requested
if "ColumnEncodingUtility" in config["utilities"]:
analyze_schema_compression.configure(
configDetail["outputFile"], configDetail["db"],
configDetail["dbUser"], usePassword, configDetail["dbHost"],
configDetail["dbPort"], configDetail["analyzeSchema"],
configDetail["targetSchema"], configDetail["analyzeTable"], 1,
True, configDetail["querySlotCount"], configDetail["ignoreErrors"],
configDetail["force"], configDetail["dropOldData"],
configDetail["comprows"], configDetail["queryGroup"],
configDetail["debug"])
encoding_result = analyze_schema_compression.run()
print "Processing Complete"
return encoding_result
def event_handler(event, context):
current_region = 'us-east-1'
if region_key not in os.environ:
print("Warning - using default region %s" % current_region)
else:
current_region = os.environ[region_key]
kms_connection = boto3.client('kms', region_name=current_region)
# load the configuration file
config_location = LOCAL_CONFIG
if event is not None and 'ConfigLocation' in event:
config_location = event['ConfigLocation']
if config_location.startswith("s3://"):
# load the configuration file from S3
s3_client = boto3.client('s3', region_name=current_region)
bucket = config_location.replace('s3://', '').split("/")[0]
key = config_location.replace('s3://' + bucket + "/", '')
obj = s3_client.get_object(Bucket=bucket, Key=key)
config_body = obj['Body'].read()
config = json.loads(config_body)
elif config_location == LOCAL_CONFIG:
# load from the local configuration
if not os.path.isfile("config.json"):
raise Exception("Unable to resolve local config.json file")
else:
config_file = open("config.json", 'r')
config = json.load(config_file)
if config is None:
raise Exception("No Configuration Found")
else:
raise Exception("Unsupported configuration location %s" %
config_location)
config_detail = config["configuration"]
# convert the provided configuration into something that the utilities we're calling will understand
config_detail = config_constants.normalise_config(config_detail)
if config_constants.DEBUG in config_detail and config_detail[
config_constants.DEBUG]:
debug = True
if debug:
print("Using Provided Configuration:")
print(config_detail)
# KMS crypto authorisation context
auth_context = None
if config_constants.KMS_AUTH_CONTEXT in config_detail:
auth_context = config_detail[config_constants.KMS_AUTH_CONTEXT]
# convert to json
auth_context = json.loads(auth_context)
if debug:
print("Using Authorisation Context for decryption")
print(auth_context)
# resolve password
encrypted_password = base64.b64decode(
config_detail[config_constants.ENCRYPTED_PASSWORD])
if encrypted_password != "" and encrypted_password is not None:
if auth_context is not None:
# decrypt the password using KMS
use_password = kms_connection.decrypt(
CiphertextBlob=encrypted_password,
EncryptionContext=auth_context)['Plaintext']
else:
# decrypt the password using KMS
use_password = kms_connection.decrypt(
CiphertextBlob=encrypted_password)['Plaintext']
else:
raise Exception(
"Unable to run Utilities without a configured Password")
config_detail[config_constants.DB_PASSWORD] = use_password
run_utilities = []
if event is not None and "ExecuteUtility" in event:
if event["ExecuteUtility"] == COLUMN_ENCODING:
run_utilities.append(COLUMN_ENCODING)
elif event["ExecuteUtility"] == ANALYZE_VACUUM:
run_utilities.append(ANALYZE_VACUUM)
elif event["ExecuteUtility"] == ANALYZE:
run_utilities.append(ANALYZE)
elif event["ExecuteUtility"] == VACUUM:
run_utilities.append(VACUUM)
elif event["ExecuteUtility"] == MONITORING:
run_utilities.append(MONITORING)
elif 'utilities' in config:
# run each utility, if requested
if COLUMN_ENCODING in config["utilities"]:
run_utilities.append(COLUMN_ENCODING)
if ANALYZE_VACUUM in config["utilities"]:
run_utilities.append(ANALYZE_VACUUM)
if ANALYZE in config["utilities"]:
run_utilities.append(ANALYZE)
if VACUUM in config["utilities"]:
run_utilities.append(VACUUM)
if MONITORING in config["utilities"]:
run_utilities.append(MONITORING)
else:
print("No Utilities configured to run. Exiting!")
return
results = []
for util in run_utilities:
if util == COLUMN_ENCODING:
print("Running %s" % util)
analyze_schema_compression.configure(**config_detail)
encoding_result = analyze_schema_compression.run()
results.append(encoding_result)
elif util == ANALYZE_VACUUM:
print("Running %s" % util)
analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail)
if analyze_result == 0:
results.append("OK")
elif util == ANALYZE:
print("Running %s" % util)
# turn on correct flag
config_detail[config_constants.DO_ANALYZE] = True
config_detail[config_constants.DO_VACUUM] = False
analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail)
if analyze_result == 0:
results.append("OK")
elif util == VACUUM:
print("Running %s" % util)
# turn on correct flag
config_detail[config_constants.DO_ANALYZE] = False
config_detail[config_constants.DO_VACUUM] = True
analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail)
if analyze_result == 0:
results.append("OK")
elif util == MONITORING:
print("Running %s" % util)
redshift_monitoring.monitor_cluster([config_detail, os.environ])
print("Processing Complete")
return results
def event_handler(event, context):
current_region = 'us-east-1'
if region_key not in os.environ:
print("Warning - using default region %s" % current_region)
else:
current_region = os.environ[region_key]
kms_connection = boto3.client('kms', region_name=current_region)
# load the configuration file
config_location = config_constants.LOCAL_CONFIG
if event is not None and 'ConfigLocation' in event:
config_location = event['ConfigLocation']
global debug
config = common.get_config(config_location, current_region, debug)
if config_constants.DEBUG in config and config[config_constants.DEBUG]:
debug = True
if debug:
print("Configuration File Contents:")
print(config)
# extract the password
use_password = common.get_password(kms_connection, config, debug)
# bind the password back into the configuration so we can pass it forward
config[config_constants.DB_PASSWORD] = use_password
run_utilities = []
if event is not None and "ExecuteUtility" in event:
if event["ExecuteUtility"] == config_constants.COLUMN_ENCODING:
run_utilities.append(config_constants.COLUMN_ENCODING)
elif event["ExecuteUtility"] == config_constants.ANALYZE_VACUUM:
run_utilities.append(config_constants.ANALYZE_VACUUM)
elif event["ExecuteUtility"] == config_constants.ANALYZE:
run_utilities.append(config_constants.ANALYZE)
elif event["ExecuteUtility"] == config_constants.VACUUM:
run_utilities.append(config_constants.VACUUM)
elif event["ExecuteUtility"] == config_constants.MONITORING:
run_utilities.append(config_constants.MONITORING)
elif event["ExecuteUtility"] == config_constants.TABLE_PERSISTENCE:
run_utilities.append(config_constants.TABLE_PERSISTENCE)
elif event["ExecuteUtility"] == config_constants.WLM_SCHEDULER:
run_utilities.append(config_constants.WLM_SCHEDULER)
elif 'utilities' in config:
# run each utility, if requested
if config_constants.COLUMN_ENCODING in config["utilities"]:
run_utilities.append(config_constants.COLUMN_ENCODING)
if config_constants.ANALYZE_VACUUM in config["utilities"]:
run_utilities.append(config_constants.ANALYZE_VACUUM)
if config_constants.ANALYZE in config["utilities"]:
run_utilities.append(config_constants.ANALYZE)
if config_constants.VACUUM in config["utilities"]:
run_utilities.append(config_constants.VACUUM)
if config_constants.MONITORING in config["utilities"]:
run_utilities.append(config_constants.MONITORING)
if config_constants.TABLE_PERSISTENCE in config["utilities"]:
run_utilities.append(config_constants.TABLE_PERSISTENCE)
if config_constants.WLM_SCHEDULER in config["utilities"]:
run_utilities.append(config_constants.WLM_SCHEDULER)
else:
print("No Utilities configured to run. Exiting!")
return
results = []
for util in run_utilities:
if util == config_constants.COLUMN_ENCODING:
print("Running %s" % util)
analyze_schema_compression.configure(**config)
encoding_result = analyze_schema_compression.run()
results.append(encoding_result)
elif util == config_constants.ANALYZE_VACUUM:
print("Running %s" % util)
analyze_result = analyze_vacuum.run_analyze_vacuum(**config)
if analyze_result == 0:
results.append("OK")
elif util == config_constants.ANALYZE:
print("Running %s" % util)
# turn on correct flag
config[config_constants.DO_ANALYZE] = True
config[config_constants.DO_VACUUM] = False
analyze_result = analyze_vacuum.run_analyze_vacuum(**config)
if analyze_result == 0:
results.append("OK")
elif util == config_constants.VACUUM:
print("Running %s" % util)
# turn on correct flag
config[config_constants.DO_ANALYZE] = False
config[config_constants.DO_VACUUM] = True
analyze_result = analyze_vacuum.run_analyze_vacuum(**config)
if analyze_result == 0:
results.append("OK")
elif util == config_constants.MONITORING:
print("Running %s" % util)
redshift_monitoring.monitor_cluster([config, os.environ])
elif util == config_constants.TABLE_PERSISTENCE:
print("Running %s" % util)
snapshot_system_stats.snapshot([config, os.environ])
elif util == config_constants.WLM_SCHEDULER:
print("Running %s" % util)
wlm_scheduler.run_scheduler(config)
print("Processing Complete")
return results
def event_handler(event, context):
currentRegion = 'us-east-1'
try:
currentRegion = os.environ[region_key]
if currentRegion is None or currentRegion == '':
raise KeyError
except KeyError:
raise Exception("Unable to resolve environment variable %s" %
region_key)
kmsConnection = boto3.client('kms', region_name=currentRegion)
# KMS crypto authorisation context
authContext = utils.get_encryption_context(currentRegion)
# load the configuration file
config_location = LOCAL_CONFIG
if 'ConfigLocation' in event:
config_location = event['ConfigLocation']
if config_location.startswith("s3://"):
s3client = boto3.client('s3', region_name=currentRegion)
bucket = config_location.replace('s3://', '').split("/")[0]
key = config_location.replace('s3://' + bucket + "/", '')
obj = s3client.get_object(Bucket=bucket, Key=key)
config = json.loads(obj['Body'].read())
elif config_location == LOCAL_CONFIG:
try:
config_file = open("config.json", 'r')
config = json.load(config_file)
if config is None:
raise Exception("No Configuration Found")
except:
print(sys.exc_info()[0])
raise
else:
raise Exception("Unsupported configuration location %s" %
config_location)
# resolve password
configDetail = config["configuration"]
encryptedPassword = configDetail["dbPassword"]
encryptedPassword = base64.b64decode(encryptedPassword)
if encryptedPassword != "" and encryptedPassword is not None:
# decrypt the password using KMS
usePassword = kmsConnection.decrypt(
CiphertextBlob=encryptedPassword,
EncryptionContext=authContext)['Plaintext']
else:
raise Exception(
"Unable to run Utilities without a configured Password")
run_utilities = []
if "ExecuteUtility" in event:
if event["ExecuteUtility"] == COLUMN_ENCODING:
run_utilities.append(COLUMN_ENCODING)
elif event["ExecuteUtility"] == ANALYZE_VACUUM:
run_utilities.append(ANALYZE_VACUUM)
elif 'utilities' in config:
# run each utility, if requested
if COLUMN_ENCODING in config["utilities"]:
run_utilities.append(COLUMN_ENCODING)
if COLUMN_ENCODING in config["utilities"]:
run_utilities.append(ANALYZE_VACUUM)
else:
print("No Utilities configured to run. Exiting!")
return
results = []
for util in run_utilities:
if util == COLUMN_ENCODING:
print("Running %s" % util)
analyze_schema_compression.configure(
configDetail["outputFile"], configDetail["db"],
configDetail["dbUser"], usePassword, configDetail["dbHost"],
configDetail["dbPort"], configDetail["analyzeSchema"],
configDetail["targetSchema"], configDetail["analyzeTable"],
configDetail["analyze_col_width"], configDetail["threads"],
configDetail["do-execute"], configDetail["querySlotCount"],
configDetail["ignoreErrors"], configDetail["force"],
configDetail["dropOldData"], configDetail["comprows"],
configDetail["queryGroup"], configDetail["debug"],
configDetail["ssl-option"], None)
encoding_result = analyze_schema_compression.run()
results.append(encoding_result)
elif util == ANALYZE_VACUUM:
print("Running %s" % util)
analyze_result = analyze_vacuum.run_analyze_vacuum(
configDetail["dbHost"], configDetail["dbPort"],
configDetail["dbUser"], usePassword, configDetail["db"],
configDetail["queryGroup"], configDetail["querySlotCount"],
configDetail["doVacuum"], configDetail["doAnalyze"],
configDetail["analyzeSchema"], configDetail["analyzeTable"],
configDetail["tableBlacklist"], configDetail["ignoreErrors"],
configDetail["ssl-option"])
if analyze_result == 0:
results.append("OK")
print("Processing Complete")
return results
def event_handler(event, context):
current_region = 'us-east-1'
if region_key not in os.environ:
print("Warning - using default region %s" % current_region)
else:
current_region = os.environ[region_key]
kms_connection = boto3.client('kms', region_name=current_region)
# load the configuration file
config_location = config_constants.LOCAL_CONFIG
if event is not None and 'ConfigLocation' in event:
config_location = event['ConfigLocation']
global debug
config = common.get_config(config_location, current_region, debug)
if config_constants.DEBUG in config and config[config_constants.DEBUG]:
debug = True
if debug:
print("Configuration File Contents:")
print(config)
# extract the password
use_password = common.get_password(kms_connection, config, debug)
# bind the password back into the configuration so we can pass it forward
config[config_constants.DB_PASSWORD] = use_password
run_utilities = []
if event is not None and "ExecuteUtility" in event:
if event["ExecuteUtility"] == config_constants.COLUMN_ENCODING:
run_utilities.append(config_constants.COLUMN_ENCODING)
elif event["ExecuteUtility"] == config_constants.ANALYZE_VACUUM:
run_utilities.append(config_constants.ANALYZE_VACUUM)
elif event["ExecuteUtility"] == config_constants.ANALYZE:
run_utilities.append(config_constants.ANALYZE)
elif event["ExecuteUtility"] == config_constants.VACUUM:
run_utilities.append(config_constants.VACUUM)
elif event["ExecuteUtility"] == config_constants.MONITORING:
run_utilities.append(config_constants.MONITORING)
elif event["ExecuteUtility"] == config_constants.TABLE_PERSISTENCE:
run_utilities.append(config_constants.TABLE_PERSISTENCE)
elif event["ExecuteUtility"] == config_constants.WLM_SCHEDULER:
run_utilities.append(config_constants.WLM_SCHEDULER)
elif 'utilities' in config:
# run each utility, if requested
if config_constants.COLUMN_ENCODING in config["utilities"]:
run_utilities.append(config_constants.COLUMN_ENCODING)
if config_constants.ANALYZE_VACUUM in config["utilities"]:
run_utilities.append(config_constants.ANALYZE_VACUUM)
if config_constants.ANALYZE in config["utilities"]:
run_utilities.append(config_constants.ANALYZE)
if config_constants.VACUUM in config["utilities"]:
run_utilities.append(config_constants.VACUUM)
if config_constants.MONITORING in config["utilities"]:
run_utilities.append(config_constants.MONITORING)
if config_constants.TABLE_PERSISTENCE in config["utilities"]:
run_utilities.append(config_constants.TABLE_PERSISTENCE)
if config_constants.WLM_SCHEDULER in config["utilities"]:
run_utilities.append(config_constants.WLM_SCHEDULER)
else:
print("No Utilities configured to run. Exiting!")
return
results = []
for util in run_utilities:
if util == config_constants.COLUMN_ENCODING:
print("Running %s" % util)
analyze_schema_compression.configure(**config)
encoding_result = analyze_schema_compression.run()
results.append(encoding_result)
elif util == config_constants.ANALYZE_VACUUM:
print("Running %s" % util)
analyze_result = analyze_vacuum.run_analyze_vacuum(**config)
if analyze_result == 0:
results.append("OK")
elif util == config_constants.ANALYZE:
print("Running %s" % util)
# turn on correct flag
config[config_constants.DO_ANALYZE] = True
config[config_constants.DO_VACUUM] = False
analyze_result = analyze_vacuum.run_analyze_vacuum(**config)
if analyze_result == 0:
results.append("OK")
elif util == config_constants.VACUUM:
print("Running %s" % util)
# turn on correct flag
config[config_constants.DO_ANALYZE] = False
config[config_constants.DO_VACUUM] = True
analyze_result = analyze_vacuum.run_analyze_vacuum(**config)
if analyze_result == 0:
results.append("OK")
elif util == config_constants.MONITORING:
print("Running %s" % util)
redshift_monitoring.monitor_cluster([config, os.environ])
elif util == config_constants.TABLE_PERSISTENCE:
print("Running %s" % util)
snapshot_system_stats.snapshot([config, os.environ])
elif util == config_constants.WLM_SCHEDULER:
print("Running %s" % util)
wlm_scheduler.run_scheduler(config)
print("Processing Complete")
return results