def event_handler(event, context): try: currentRegion = os.environ[region_key] if currentRegion == None or currentRegion == '': raise KeyError except KeyError: raise Exception("Unable to resolve environment variable %s" % region_key) kmsConnection = boto3.client('kms', region_name=currentRegion) # KMS crypto authorisation context authContext = utils.get_encryption_context(currentRegion) global config # lazy load the configuration file if config == None: try: config_file = open("config.json", 'r') config = json.load(config_file) if config == None: raise Exception("No Configuration Found") except: print sys.exc_info()[0] raise # resolve password configDetail = config["configuration"] encryptedPassword = configDetail["dbPassword"] encryptedPassword = base64.b64decode(encryptedPassword) if encryptedPassword != "" and encryptedPassword != None: # decrypt the password using KMS usePassword = kmsConnection.decrypt( CiphertextBlob=encryptedPassword, EncryptionContext=authContext)['Plaintext'] else: raise Exception( "Unable to run Utilities without a configured Password") # run the column encoding utility, if requested if "ColumnEncodingUtility" in config["utilities"]: analyze_schema_compression.configure( configDetail["outputFile"], configDetail["db"], configDetail["dbUser"], usePassword, configDetail["dbHost"], configDetail["dbPort"], configDetail["analyzeSchema"], configDetail["targetSchema"], configDetail["analyzeTable"], 1, True, configDetail["querySlotCount"], configDetail["ignoreErrors"], configDetail["force"], configDetail["dropOldData"], configDetail["comprows"], configDetail["queryGroup"], configDetail["debug"]) encoding_result = analyze_schema_compression.run() print "Processing Complete" return encoding_result
def event_handler(event, context): current_region = 'us-east-1' if region_key not in os.environ: print("Warning - using default region %s" % current_region) else: current_region = os.environ[region_key] kms_connection = boto3.client('kms', region_name=current_region) # load the configuration file config_location = LOCAL_CONFIG if event is not None and 'ConfigLocation' in event: config_location = event['ConfigLocation'] if config_location.startswith("s3://"): # load the configuration file from S3 s3_client = boto3.client('s3', region_name=current_region) bucket = config_location.replace('s3://', '').split("/")[0] key = config_location.replace('s3://' + bucket + "/", '') obj = s3_client.get_object(Bucket=bucket, Key=key) config_body = obj['Body'].read() config = json.loads(config_body) elif config_location == LOCAL_CONFIG: # load from the local configuration if not os.path.isfile("config.json"): raise Exception("Unable to resolve local config.json file") else: config_file = open("config.json", 'r') config = json.load(config_file) if config is None: raise Exception("No Configuration Found") else: raise Exception("Unsupported configuration location %s" % config_location) config_detail = config["configuration"] # convert the provided configuration into something that the utilities we're calling will understand config_detail = config_constants.normalise_config(config_detail) if config_constants.DEBUG in config_detail and config_detail[ config_constants.DEBUG]: debug = True if debug: print("Using Provided Configuration:") print(config_detail) # KMS crypto authorisation context auth_context = None if config_constants.KMS_AUTH_CONTEXT in config_detail: auth_context = config_detail[config_constants.KMS_AUTH_CONTEXT] # convert to json auth_context = json.loads(auth_context) if debug: print("Using Authorisation Context for decryption") print(auth_context) # resolve password encrypted_password = base64.b64decode( config_detail[config_constants.ENCRYPTED_PASSWORD]) if encrypted_password != "" and encrypted_password is not None: if auth_context is not None: # decrypt the password using KMS use_password = kms_connection.decrypt( CiphertextBlob=encrypted_password, EncryptionContext=auth_context)['Plaintext'] else: # decrypt the password using KMS use_password = kms_connection.decrypt( CiphertextBlob=encrypted_password)['Plaintext'] else: raise Exception( "Unable to run Utilities without a configured Password") config_detail[config_constants.DB_PASSWORD] = use_password run_utilities = [] if event is not None and "ExecuteUtility" in event: if event["ExecuteUtility"] == COLUMN_ENCODING: run_utilities.append(COLUMN_ENCODING) elif event["ExecuteUtility"] == ANALYZE_VACUUM: run_utilities.append(ANALYZE_VACUUM) elif event["ExecuteUtility"] == ANALYZE: run_utilities.append(ANALYZE) elif event["ExecuteUtility"] == VACUUM: run_utilities.append(VACUUM) elif event["ExecuteUtility"] == MONITORING: run_utilities.append(MONITORING) elif 'utilities' in config: # run each utility, if requested if COLUMN_ENCODING in config["utilities"]: run_utilities.append(COLUMN_ENCODING) if ANALYZE_VACUUM in config["utilities"]: run_utilities.append(ANALYZE_VACUUM) if ANALYZE in config["utilities"]: run_utilities.append(ANALYZE) if VACUUM in config["utilities"]: run_utilities.append(VACUUM) if MONITORING in config["utilities"]: run_utilities.append(MONITORING) else: print("No Utilities configured to run. Exiting!") return results = [] for util in run_utilities: if util == COLUMN_ENCODING: print("Running %s" % util) analyze_schema_compression.configure(**config_detail) encoding_result = analyze_schema_compression.run() results.append(encoding_result) elif util == ANALYZE_VACUUM: print("Running %s" % util) analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail) if analyze_result == 0: results.append("OK") elif util == ANALYZE: print("Running %s" % util) # turn on correct flag config_detail[config_constants.DO_ANALYZE] = True config_detail[config_constants.DO_VACUUM] = False analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail) if analyze_result == 0: results.append("OK") elif util == VACUUM: print("Running %s" % util) # turn on correct flag config_detail[config_constants.DO_ANALYZE] = False config_detail[config_constants.DO_VACUUM] = True analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail) if analyze_result == 0: results.append("OK") elif util == MONITORING: print("Running %s" % util) redshift_monitoring.monitor_cluster([config_detail, os.environ]) print("Processing Complete") return results
def event_handler(event, context): current_region = 'us-east-1' if region_key not in os.environ: print("Warning - using default region %s" % current_region) else: current_region = os.environ[region_key] kms_connection = boto3.client('kms', region_name=current_region) # load the configuration file config_location = config_constants.LOCAL_CONFIG if event is not None and 'ConfigLocation' in event: config_location = event['ConfigLocation'] global debug config = common.get_config(config_location, current_region, debug) if config_constants.DEBUG in config and config[config_constants.DEBUG]: debug = True if debug: print("Configuration File Contents:") print(config) # extract the password use_password = common.get_password(kms_connection, config, debug) # bind the password back into the configuration so we can pass it forward config[config_constants.DB_PASSWORD] = use_password run_utilities = [] if event is not None and "ExecuteUtility" in event: if event["ExecuteUtility"] == config_constants.COLUMN_ENCODING: run_utilities.append(config_constants.COLUMN_ENCODING) elif event["ExecuteUtility"] == config_constants.ANALYZE_VACUUM: run_utilities.append(config_constants.ANALYZE_VACUUM) elif event["ExecuteUtility"] == config_constants.ANALYZE: run_utilities.append(config_constants.ANALYZE) elif event["ExecuteUtility"] == config_constants.VACUUM: run_utilities.append(config_constants.VACUUM) elif event["ExecuteUtility"] == config_constants.MONITORING: run_utilities.append(config_constants.MONITORING) elif event["ExecuteUtility"] == config_constants.TABLE_PERSISTENCE: run_utilities.append(config_constants.TABLE_PERSISTENCE) elif event["ExecuteUtility"] == config_constants.WLM_SCHEDULER: run_utilities.append(config_constants.WLM_SCHEDULER) elif 'utilities' in config: # run each utility, if requested if config_constants.COLUMN_ENCODING in config["utilities"]: run_utilities.append(config_constants.COLUMN_ENCODING) if config_constants.ANALYZE_VACUUM in config["utilities"]: run_utilities.append(config_constants.ANALYZE_VACUUM) if config_constants.ANALYZE in config["utilities"]: run_utilities.append(config_constants.ANALYZE) if config_constants.VACUUM in config["utilities"]: run_utilities.append(config_constants.VACUUM) if config_constants.MONITORING in config["utilities"]: run_utilities.append(config_constants.MONITORING) if config_constants.TABLE_PERSISTENCE in config["utilities"]: run_utilities.append(config_constants.TABLE_PERSISTENCE) if config_constants.WLM_SCHEDULER in config["utilities"]: run_utilities.append(config_constants.WLM_SCHEDULER) else: print("No Utilities configured to run. Exiting!") return results = [] for util in run_utilities: if util == config_constants.COLUMN_ENCODING: print("Running %s" % util) analyze_schema_compression.configure(**config) encoding_result = analyze_schema_compression.run() results.append(encoding_result) elif util == config_constants.ANALYZE_VACUUM: print("Running %s" % util) analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.ANALYZE: print("Running %s" % util) # turn on correct flag config[config_constants.DO_ANALYZE] = True config[config_constants.DO_VACUUM] = False analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.VACUUM: print("Running %s" % util) # turn on correct flag config[config_constants.DO_ANALYZE] = False config[config_constants.DO_VACUUM] = True analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.MONITORING: print("Running %s" % util) redshift_monitoring.monitor_cluster([config, os.environ]) elif util == config_constants.TABLE_PERSISTENCE: print("Running %s" % util) snapshot_system_stats.snapshot([config, os.environ]) elif util == config_constants.WLM_SCHEDULER: print("Running %s" % util) wlm_scheduler.run_scheduler(config) print("Processing Complete") return results
def event_handler(event, context): currentRegion = 'us-east-1' try: currentRegion = os.environ[region_key] if currentRegion is None or currentRegion == '': raise KeyError except KeyError: raise Exception("Unable to resolve environment variable %s" % region_key) kmsConnection = boto3.client('kms', region_name=currentRegion) # KMS crypto authorisation context authContext = utils.get_encryption_context(currentRegion) # load the configuration file config_location = LOCAL_CONFIG if 'ConfigLocation' in event: config_location = event['ConfigLocation'] if config_location.startswith("s3://"): s3client = boto3.client('s3', region_name=currentRegion) bucket = config_location.replace('s3://', '').split("/")[0] key = config_location.replace('s3://' + bucket + "/", '') obj = s3client.get_object(Bucket=bucket, Key=key) config = json.loads(obj['Body'].read()) elif config_location == LOCAL_CONFIG: try: config_file = open("config.json", 'r') config = json.load(config_file) if config is None: raise Exception("No Configuration Found") except: print(sys.exc_info()[0]) raise else: raise Exception("Unsupported configuration location %s" % config_location) # resolve password configDetail = config["configuration"] encryptedPassword = configDetail["dbPassword"] encryptedPassword = base64.b64decode(encryptedPassword) if encryptedPassword != "" and encryptedPassword is not None: # decrypt the password using KMS usePassword = kmsConnection.decrypt( CiphertextBlob=encryptedPassword, EncryptionContext=authContext)['Plaintext'] else: raise Exception( "Unable to run Utilities without a configured Password") run_utilities = [] if "ExecuteUtility" in event: if event["ExecuteUtility"] == COLUMN_ENCODING: run_utilities.append(COLUMN_ENCODING) elif event["ExecuteUtility"] == ANALYZE_VACUUM: run_utilities.append(ANALYZE_VACUUM) elif 'utilities' in config: # run each utility, if requested if COLUMN_ENCODING in config["utilities"]: run_utilities.append(COLUMN_ENCODING) if COLUMN_ENCODING in config["utilities"]: run_utilities.append(ANALYZE_VACUUM) else: print("No Utilities configured to run. Exiting!") return results = [] for util in run_utilities: if util == COLUMN_ENCODING: print("Running %s" % util) analyze_schema_compression.configure( configDetail["outputFile"], configDetail["db"], configDetail["dbUser"], usePassword, configDetail["dbHost"], configDetail["dbPort"], configDetail["analyzeSchema"], configDetail["targetSchema"], configDetail["analyzeTable"], configDetail["analyze_col_width"], configDetail["threads"], configDetail["do-execute"], configDetail["querySlotCount"], configDetail["ignoreErrors"], configDetail["force"], configDetail["dropOldData"], configDetail["comprows"], configDetail["queryGroup"], configDetail["debug"], configDetail["ssl-option"], None) encoding_result = analyze_schema_compression.run() results.append(encoding_result) elif util == ANALYZE_VACUUM: print("Running %s" % util) analyze_result = analyze_vacuum.run_analyze_vacuum( configDetail["dbHost"], configDetail["dbPort"], configDetail["dbUser"], usePassword, configDetail["db"], configDetail["queryGroup"], configDetail["querySlotCount"], configDetail["doVacuum"], configDetail["doAnalyze"], configDetail["analyzeSchema"], configDetail["analyzeTable"], configDetail["tableBlacklist"], configDetail["ignoreErrors"], configDetail["ssl-option"]) if analyze_result == 0: results.append("OK") print("Processing Complete") return results
def event_handler(event, context): current_region = 'us-east-1' if region_key not in os.environ: print("Warning - using default region %s" % current_region) else: current_region = os.environ[region_key] kms_connection = boto3.client('kms', region_name=current_region) # load the configuration file config_location = config_constants.LOCAL_CONFIG if event is not None and 'ConfigLocation' in event: config_location = event['ConfigLocation'] global debug config = common.get_config(config_location, current_region, debug) if config_constants.DEBUG in config and config[config_constants.DEBUG]: debug = True if debug: print("Configuration File Contents:") print(config) # extract the password use_password = common.get_password(kms_connection, config, debug) # bind the password back into the configuration so we can pass it forward config[config_constants.DB_PASSWORD] = use_password run_utilities = [] if event is not None and "ExecuteUtility" in event: if event["ExecuteUtility"] == config_constants.COLUMN_ENCODING: run_utilities.append(config_constants.COLUMN_ENCODING) elif event["ExecuteUtility"] == config_constants.ANALYZE_VACUUM: run_utilities.append(config_constants.ANALYZE_VACUUM) elif event["ExecuteUtility"] == config_constants.ANALYZE: run_utilities.append(config_constants.ANALYZE) elif event["ExecuteUtility"] == config_constants.VACUUM: run_utilities.append(config_constants.VACUUM) elif event["ExecuteUtility"] == config_constants.MONITORING: run_utilities.append(config_constants.MONITORING) elif event["ExecuteUtility"] == config_constants.TABLE_PERSISTENCE: run_utilities.append(config_constants.TABLE_PERSISTENCE) elif event["ExecuteUtility"] == config_constants.WLM_SCHEDULER: run_utilities.append(config_constants.WLM_SCHEDULER) elif 'utilities' in config: # run each utility, if requested if config_constants.COLUMN_ENCODING in config["utilities"]: run_utilities.append(config_constants.COLUMN_ENCODING) if config_constants.ANALYZE_VACUUM in config["utilities"]: run_utilities.append(config_constants.ANALYZE_VACUUM) if config_constants.ANALYZE in config["utilities"]: run_utilities.append(config_constants.ANALYZE) if config_constants.VACUUM in config["utilities"]: run_utilities.append(config_constants.VACUUM) if config_constants.MONITORING in config["utilities"]: run_utilities.append(config_constants.MONITORING) if config_constants.TABLE_PERSISTENCE in config["utilities"]: run_utilities.append(config_constants.TABLE_PERSISTENCE) if config_constants.WLM_SCHEDULER in config["utilities"]: run_utilities.append(config_constants.WLM_SCHEDULER) else: print("No Utilities configured to run. Exiting!") return results = [] for util in run_utilities: if util == config_constants.COLUMN_ENCODING: print("Running %s" % util) analyze_schema_compression.configure(**config) encoding_result = analyze_schema_compression.run() results.append(encoding_result) elif util == config_constants.ANALYZE_VACUUM: print("Running %s" % util) analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.ANALYZE: print("Running %s" % util) # turn on correct flag config[config_constants.DO_ANALYZE] = True config[config_constants.DO_VACUUM] = False analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.VACUUM: print("Running %s" % util) # turn on correct flag config[config_constants.DO_ANALYZE] = False config[config_constants.DO_VACUUM] = True analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.MONITORING: print("Running %s" % util) redshift_monitoring.monitor_cluster([config, os.environ]) elif util == config_constants.TABLE_PERSISTENCE: print("Running %s" % util) snapshot_system_stats.snapshot([config, os.environ]) elif util == config_constants.WLM_SCHEDULER: print("Running %s" % util) wlm_scheduler.run_scheduler(config) print("Processing Complete") return results