def main(argv): # get environmental args args = {config_constants.DB_NAME: os.environ.get('PGDATABASE', None), config_constants.DB_USER: os.environ.get('PGUSER', None), config_constants.DB_PASSWORD: os.environ.get('PGPASSWORD', None), config_constants.DB_HOST: os.environ.get('PGHOST', None), config_constants.DB_PORT: os.environ.get('PGPORT', 5439)} # add argparse args args.update(parse_args) if args.get(config_constants.OUTPUT_FILE) is not None: sys.stdout = open(args.get(config_constants.OUTPUT_FILE), 'w') # invoke the main method of the utility result = analyze_vacuum.run_analyze_vacuum(**args) if result is not None: sys.exit(result) else: sys.exit(0)
def event_handler(event, context): current_region = 'us-east-1' if region_key not in os.environ: print("Warning - using default region %s" % current_region) else: current_region = os.environ[region_key] kms_connection = boto3.client('kms', region_name=current_region) # load the configuration file config_location = LOCAL_CONFIG if event is not None and 'ConfigLocation' in event: config_location = event['ConfigLocation'] if config_location.startswith("s3://"): # load the configuration file from S3 s3_client = boto3.client('s3', region_name=current_region) bucket = config_location.replace('s3://', '').split("/")[0] key = config_location.replace('s3://' + bucket + "/", '') obj = s3_client.get_object(Bucket=bucket, Key=key) config_body = obj['Body'].read() config = json.loads(config_body) elif config_location == LOCAL_CONFIG: # load from the local configuration if not os.path.isfile("config.json"): raise Exception("Unable to resolve local config.json file") else: config_file = open("config.json", 'r') config = json.load(config_file) if config is None: raise Exception("No Configuration Found") else: raise Exception("Unsupported configuration location %s" % config_location) config_detail = config["configuration"] # convert the provided configuration into something that the utilities we're calling will understand config_detail = config_constants.normalise_config(config_detail) if config_constants.DEBUG in config_detail and config_detail[ config_constants.DEBUG]: debug = True if debug: print("Using Provided Configuration:") print(config_detail) # KMS crypto authorisation context auth_context = None if config_constants.KMS_AUTH_CONTEXT in config_detail: auth_context = config_detail[config_constants.KMS_AUTH_CONTEXT] # convert to json auth_context = json.loads(auth_context) if debug: print("Using Authorisation Context for decryption") print(auth_context) # resolve password encrypted_password = base64.b64decode( config_detail[config_constants.ENCRYPTED_PASSWORD]) if encrypted_password != "" and encrypted_password is not None: if auth_context is not None: # decrypt the password using KMS use_password = kms_connection.decrypt( CiphertextBlob=encrypted_password, EncryptionContext=auth_context)['Plaintext'] else: # decrypt the password using KMS use_password = kms_connection.decrypt( CiphertextBlob=encrypted_password)['Plaintext'] else: raise Exception( "Unable to run Utilities without a configured Password") config_detail[config_constants.DB_PASSWORD] = use_password run_utilities = [] if event is not None and "ExecuteUtility" in event: if event["ExecuteUtility"] == COLUMN_ENCODING: run_utilities.append(COLUMN_ENCODING) elif event["ExecuteUtility"] == ANALYZE_VACUUM: run_utilities.append(ANALYZE_VACUUM) elif event["ExecuteUtility"] == ANALYZE: run_utilities.append(ANALYZE) elif event["ExecuteUtility"] == VACUUM: run_utilities.append(VACUUM) elif event["ExecuteUtility"] == MONITORING: run_utilities.append(MONITORING) elif 'utilities' in config: # run each utility, if requested if COLUMN_ENCODING in config["utilities"]: run_utilities.append(COLUMN_ENCODING) if ANALYZE_VACUUM in config["utilities"]: run_utilities.append(ANALYZE_VACUUM) if ANALYZE in config["utilities"]: run_utilities.append(ANALYZE) if VACUUM in config["utilities"]: run_utilities.append(VACUUM) if MONITORING in config["utilities"]: run_utilities.append(MONITORING) else: print("No Utilities configured to run. Exiting!") return results = [] for util in run_utilities: if util == COLUMN_ENCODING: print("Running %s" % util) analyze_schema_compression.configure(**config_detail) encoding_result = analyze_schema_compression.run() results.append(encoding_result) elif util == ANALYZE_VACUUM: print("Running %s" % util) analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail) if analyze_result == 0: results.append("OK") elif util == ANALYZE: print("Running %s" % util) # turn on correct flag config_detail[config_constants.DO_ANALYZE] = True config_detail[config_constants.DO_VACUUM] = False analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail) if analyze_result == 0: results.append("OK") elif util == VACUUM: print("Running %s" % util) # turn on correct flag config_detail[config_constants.DO_ANALYZE] = False config_detail[config_constants.DO_VACUUM] = True analyze_result = analyze_vacuum.run_analyze_vacuum(**config_detail) if analyze_result == 0: results.append("OK") elif util == MONITORING: print("Running %s" % util) redshift_monitoring.monitor_cluster([config_detail, os.environ]) print("Processing Complete") return results
def main(argv): master_conn = None db_connections = {} db = get_env_var('PGDATABASE', None) db_user = get_env_var('PGUSER', None) db_pwd = get_env_var('PGPASSWORD', None) db_host = get_env_var('PGHOST', None) db_port = get_env_var('PGPORT', 5439) schema_name = 'public' table_name = None blacklisted_tables = None debug = False do_execute = False query_slot_count = 1 ignore_errors = False query_group = None analyze_flag = True vacuum_flag = True require_ssl = False supported_args = """db= db-user= db-pwd= db-host= db-port= schema-name= table-name= blacklisted-tables= require-ssl= debug= output-file= slot-count= ignore-errors= query_group= analyze-flag= vacuum-flag= vacuum-parameter= min-unsorted-pct= max-unsorted-pct= deleted-pct= stats-off-pct= max-table-size-mb= min-interleaved-skew= min-interleaved-cnt=""" # extract the command line arguments try: optlist, remaining = getopt.getopt(argv[1:], "", supported_args.split()) except getopt.GetoptError as err: print(str(err)) usage() output_file = None # parse command line arguments for arg, value in optlist: if arg == "--db": if value == '' or value is None: usage() else: db = value elif arg == "--db-user": if value == '' or value is None: usage() else: db_user = value elif arg == "--db-pwd": if value == '' or value is None: usage() else: db_pwd = value elif arg == "--db-host": if value == '' or value is None: usage() else: db_host = value elif arg == "--db-port": if value != '' and value is not None: db_port = value elif arg == "--require-ssl": if value != '' and value != None: if value.upper() == 'TRUE' or value == '1': require_ssl = True elif arg == "--schema-name": if value != '' and value is not None: schema_name = value elif arg == "--table-name": if value != '' and value is not None: table_name = value elif arg == "--blacklisted-tables": if value != '' and value is not None: blacklisted_tables = value elif arg == "--debug": if value.upper() == 'TRUE': debug = True else: debug = False elif arg == "--output-file": output_file = value elif arg == "--ignore-errors": if value.upper() == 'TRUE': ignore_errors = True else: ignore_errors = False elif arg == "--slot-count": query_slot_count = int(value) elif arg == "--query_group": if value != '' and value is not None: query_group = value elif arg == "--vacuum-flag": if value.upper() == 'FALSE': vacuum_flag = False elif arg == "--analyze-flag": if value.upper() == 'FALSE': analyze_flag = False elif arg == "--vacuum-parameter": if value.upper() == 'SORT ONLY' or value.upper( ) == 'DELETE ONLY' or value.upper() == 'REINDEX': vacuum_parameter = value else: vacuum_parameter = 'FULL' elif arg == "--min-unsorted-pct": if value != '' and value is not None: min_unsorted_pct = value elif arg == "--max-unsorted-pct": if value != '' and value is not None: max_unsorted_pct = value elif arg == "--deleted-pct": if value != '' and value is not None: deleted_pct = value elif arg == "--stats-off-pct": if value != '' and value is not None: stats_off_pct = value elif arg == "--predicate-cols": if value.upper() == 'TRUE': predicate_cols = True else: predicate_cols = False elif arg == "--max-table-size-mb": if value != '' and value is not None: max_table_size_mb = value elif arg == "--min-interleaved-skew": if value != '' and value is not None: min_interleaved_skew = value elif arg == "--min-interleaved-cnt": if value != '' and value is not None: min_interleaved_cnt = value else: assert False, "Unsupported Argument " + arg usage() # Validate that we've got all the args needed if db is None: usage("Missing Parameter 'db'") if db_user is None: usage("Missing Parameter 'db-user'") if db_pwd is None: usage("Missing Parameter 'db-pwd'") if db_host is None: usage("Missing Parameter 'db-host'") if db_port is None: usage("Missing Parameter 'db-port'") if (output_file is not None): sys.stdout = open(output_file, 'w') # invoke the main method of the utility result = analyze_vacuum.run_analyze_vacuum(db_host, db_port, db_user, db_pwd, db, query_group, query_slot_count, vacuum_flag, analyze_flag, schema_name, table_name, blacklisted_tables, ignore_errors, require_ssl) if (result is not None): sys.exit(result) else: sys.exit(0)
def main(argv): supported_args = """db= db-user= db-pwd= db-host= db-port= schema-name= table-name= blacklisted-tables= suppress-cloudwatch= require-ssl= debug= output-file= slot-count= ignore-errors= query_group= analyze-flag= vacuum-flag= vacuum-parameter= min-unsorted-pct= max-unsorted-pct= stats-off-pct= predicate-cols= max-table-size-mb= min-interleaved-skew= min-interleaved-cnt=""" # extract the command line arguments try: optlist, remaining = getopt.getopt(argv[1:], "", supported_args.split()) except getopt.GetoptError as err: print(str(err)) usage() args = {config_constants.DB_NAME: get_env_var('PGDATABASE', None), config_constants.DB_USER: get_env_var('PGUSER', None), config_constants.DB_PASSWORD: get_env_var('PGPASSWORD', None), config_constants.DB_HOST: get_env_var('PGHOST', None), config_constants.DB_PORT: get_env_var('PGPORT', 5439)} # parse command line arguments for arg, value in optlist: if arg == "--db": if value == '': usage() else: args['db'] = value elif arg == "--db-user": if value == '': usage() else: args[config_constants.DB_USER] = value elif arg == "--db-pwd": if value == '': usage() else: args[config_constants.DB_PASSWORD] = value elif arg == "--db-host": if value != '': args[config_constants.DB_HOST] = value elif arg == "--db-port": if value != '' and value is not None: args[config_constants.DB_PORT] = int(value) elif arg == "--require-ssl": if value != '' and value is not None: if value.upper() == 'TRUE' or value == '1': args[config_constants.SSL] = True else: args[config_constants.SSL] = False elif arg == "--schema-name": if value != '' and value is not None: args[config_constants.SCHEMA_NAME] = value elif arg == "--table-name": if value != '' and value is not None: args[config_constants.TABLE_NAME] = value elif arg == "--blacklisted-tables": if value != '' and value is not None: args[config_constants.BLACKLISTED_TABLES] = value elif arg == "--debug": if value.upper() == 'TRUE' or value == '1': args[config_constants.DEBUG] = True elif arg == "--output-file": # open the supplied file path and bind it to stdout sys.stdout = open(value, 'w') elif arg == "--ignore-errors": if value.upper() == 'TRUE' or value == '1': args[config_constants.IGNORE_ERRORS] = True elif arg == "--slot-count": args[config_constants.QUERY_SLOT_COUNT] = int(value) elif arg == "--query_group": if value != '' and value is not None: args[config_constants.QUERY_GROUP] = value elif arg == "--vacuum-flag": if value.upper() == 'TRUE' or value == '1': args[config_constants.DO_VACUUM] = True else: args[config_constants.DO_VACUUM] = False elif arg == "--analyze-flag": if value.upper() == 'TRUE' or value == '1': args[config_constants.DO_ANALYZE] = True else: args[config_constants.DO_ANALYZE] = False elif arg == "--vacuum-parameter": if value.upper() == 'SORT ONLY' or value.upper() == 'DELETE ONLY' or value.upper() == 'REINDEX': args[config_constants.VACUUM_PARAMETER] = value else: args['vacuum_parameter'] = 'FULL' elif arg == "--min-unsorted-pct": if value != '' and value is not None: args[config_constants.MIN_UNSORTED_PCT] = value elif arg == "--max-unsorted-pct": if value != '' and value is not None: args[config_constants.MAX_UNSORTED_PCT] = value elif arg == "--stats-off-pct": if value != '' and value is not None: args[config_constants.STATS_OFF_PCT] = value elif arg == "--predicate-cols": if value.upper() == 'TRUE' or value == '1': args[config_constants.PREDICATE_COLS] = True else: args[config_constants.PREDICATE_COLS] = False elif arg == "--suppress-cloudwatch": if value.upper() == 'TRUE' or value == '1': args[config_constants.SUPPRESS_CLOUDWATCH] = True else: args[config_constants.SUPPRESS_CLOUDWATCH] = False elif arg == "--max-table-size-mb": if value != '' and value is not None: args[config_constants.MAX_TBL_SIZE_MB] = value elif arg == "--min-interleaved-skew": if value != '' and value is not None: args[config_constants.MIN_INTERLEAVED_SKEW] = value elif arg == "--min-interleaved-cnt": if value != '' and value is not None: args[config_constants.MIN_INTERLEAVED_COUNT] = value else: usage("Unsupported Argument " + arg) # Validate that we've got all the args needed if config_constants.DB_NAME not in args: usage("Missing Parameter 'db'") if config_constants.DB_USER not in args: usage("Missing Parameter 'db-user'") if config_constants.DB_PASSWORD not in args: usage("Missing Parameter 'db-pwd'") if config_constants.DB_HOST not in args: usage("Missing Parameter 'db-host'") if config_constants.DB_PORT not in args: usage("Missing Parameter 'db-port'") if config_constants.OUTPUT_FILE in args: sys.stdout = open(args['output_file'], 'w') # invoke the main method of the utility result = analyze_vacuum.run_analyze_vacuum(**args) if result is not None: sys.exit(result) else: sys.exit(0)
def event_handler(event, context): currentRegion = 'us-east-1' try: currentRegion = os.environ[region_key] if currentRegion is None or currentRegion == '': raise KeyError except KeyError: raise Exception("Unable to resolve environment variable %s" % region_key) kmsConnection = boto3.client('kms', region_name=currentRegion) # KMS crypto authorisation context authContext = utils.get_encryption_context(currentRegion) # load the configuration file config_location = LOCAL_CONFIG if 'ConfigLocation' in event: config_location = event['ConfigLocation'] if config_location.startswith("s3://"): s3client = boto3.client('s3', region_name=currentRegion) bucket = config_location.replace('s3://', '').split("/")[0] key = config_location.replace('s3://' + bucket + "/", '') obj = s3client.get_object(Bucket=bucket, Key=key) config = json.loads(obj['Body'].read()) elif config_location == LOCAL_CONFIG: try: config_file = open("config.json", 'r') config = json.load(config_file) if config is None: raise Exception("No Configuration Found") except: print(sys.exc_info()[0]) raise else: raise Exception("Unsupported configuration location %s" % config_location) # resolve password configDetail = config["configuration"] encryptedPassword = configDetail["dbPassword"] encryptedPassword = base64.b64decode(encryptedPassword) if encryptedPassword != "" and encryptedPassword is not None: # decrypt the password using KMS usePassword = kmsConnection.decrypt( CiphertextBlob=encryptedPassword, EncryptionContext=authContext)['Plaintext'] else: raise Exception( "Unable to run Utilities without a configured Password") run_utilities = [] if "ExecuteUtility" in event: if event["ExecuteUtility"] == COLUMN_ENCODING: run_utilities.append(COLUMN_ENCODING) elif event["ExecuteUtility"] == ANALYZE_VACUUM: run_utilities.append(ANALYZE_VACUUM) elif 'utilities' in config: # run each utility, if requested if COLUMN_ENCODING in config["utilities"]: run_utilities.append(COLUMN_ENCODING) if COLUMN_ENCODING in config["utilities"]: run_utilities.append(ANALYZE_VACUUM) else: print("No Utilities configured to run. Exiting!") return results = [] for util in run_utilities: if util == COLUMN_ENCODING: print("Running %s" % util) analyze_schema_compression.configure( configDetail["outputFile"], configDetail["db"], configDetail["dbUser"], usePassword, configDetail["dbHost"], configDetail["dbPort"], configDetail["analyzeSchema"], configDetail["targetSchema"], configDetail["analyzeTable"], configDetail["analyze_col_width"], configDetail["threads"], configDetail["do-execute"], configDetail["querySlotCount"], configDetail["ignoreErrors"], configDetail["force"], configDetail["dropOldData"], configDetail["comprows"], configDetail["queryGroup"], configDetail["debug"], configDetail["ssl-option"], None) encoding_result = analyze_schema_compression.run() results.append(encoding_result) elif util == ANALYZE_VACUUM: print("Running %s" % util) analyze_result = analyze_vacuum.run_analyze_vacuum( configDetail["dbHost"], configDetail["dbPort"], configDetail["dbUser"], usePassword, configDetail["db"], configDetail["queryGroup"], configDetail["querySlotCount"], configDetail["doVacuum"], configDetail["doAnalyze"], configDetail["analyzeSchema"], configDetail["analyzeTable"], configDetail["tableBlacklist"], configDetail["ignoreErrors"], configDetail["ssl-option"]) if analyze_result == 0: results.append("OK") print("Processing Complete") return results
def main(argv): supported_args = """db= db-user= db-pwd= db-host= db-port= schema-name= table-name= blacklisted-tables= suppress-cloudwatch= require-ssl= debug= output-file= slot-count= ignore-errors= query_group= analyze-flag= vacuum-flag= vacuum-parameter= min-unsorted-pct= max-unsorted-pct= stats-off-pct= predicate-cols= max-table-size-mb= min-interleaved-skew= min-interleaved-cnt=""" """ Load connection environment from secret manager """ redshift_secrets_dict = ast.literal_eval(_get_secret('bi/redshift_prod')) # extract the command line arguments try: optlist, remaining = getopt.getopt(argv[1:], "", supported_args.split()) except getopt.GetoptError as err: print(str(err)) usage() args = {config_constants.DB_NAME: redshift_secrets_dict["redshift_db"], config_constants.DB_USER: redshift_secrets_dict["redshift_user"], config_constants.DB_PASSWORD: redshift_secrets_dict["redshift_password"], config_constants.DB_HOST: redshift_secrets_dict["redshift_endpoint"], config_constants.DB_PORT: redshift_secrets_dict["redshift_port"]} # parse command line arguments for arg, value in optlist: if arg == "--db": if value == '': usage() else: args[config_constants.DB_NAME] = value elif arg == "--db-user": if value == '': usage() else: args[config_constants.DB_USER] = value elif arg == "--db-pwd": if value == '': usage() else: args[config_constants.DB_PASSWORD] = value elif arg == "--db-host": if value != '': args[config_constants.DB_HOST] = value elif arg == "--db-port": if value != '' and value is not None: args[config_constants.DB_PORT] = int(value) elif arg == "--require-ssl": if value != '' and value is not None: if value.upper() == 'TRUE' or value == '1': args[config_constants.SSL] = True else: args[config_constants.SSL] = False elif arg == "--schema-name": if value != '' and value is not None: args[config_constants.SCHEMA_NAME] = value elif arg == "--table-name": if value != '' and value is not None: args[config_constants.TABLE_NAME] = value elif arg == "--blacklisted-tables": if value != '' and value is not None: args[config_constants.BLACKLISTED_TABLES] = value elif arg == "--debug": if value.upper() == 'TRUE' or value == '1': args[config_constants.DEBUG] = True elif arg == "--output-file": # open the supplied file path and bind it to stdout sys.stdout = open(value, 'w') elif arg == "--ignore-errors": if value.upper() == 'TRUE' or value == '1': args[config_constants.IGNORE_ERRORS] = True elif arg == "--slot-count": args[config_constants.QUERY_SLOT_COUNT] = int(value) elif arg == "--query_group": if value != '' and value is not None: args[config_constants.QUERY_GROUP] = value elif arg == "--vacuum-flag": if value.upper() == 'TRUE' or value == '1': args[config_constants.DO_VACUUM] = True else: args[config_constants.DO_VACUUM] = False elif arg == "--analyze-flag": if value.upper() == 'TRUE' or value == '1': args[config_constants.DO_ANALYZE] = True else: args[config_constants.DO_ANALYZE] = False elif arg == "--vacuum-parameter": if value.upper() == 'SORT ONLY' or value.upper() == 'DELETE ONLY' or value.upper() == 'REINDEX': args[config_constants.VACUUM_PARAMETER] = value else: args['vacuum_parameter'] = 'FULL' elif arg == "--min-unsorted-pct": if value != '' and value is not None: args[config_constants.MIN_UNSORTED_PCT] = value elif arg == "--max-unsorted-pct": if value != '' and value is not None: args[config_constants.MAX_UNSORTED_PCT] = value elif arg == "--stats-off-pct": if value != '' and value is not None: args[config_constants.STATS_OFF_PCT] = value elif arg == "--predicate-cols": if value.upper() == 'TRUE' or value == '1': args[config_constants.PREDICATE_COLS] = True else: args[config_constants.PREDICATE_COLS] = False elif arg == "--suppress-cloudwatch": if value.upper() == 'TRUE' or value == '1': args[config_constants.SUPPRESS_CLOUDWATCH] = True else: args[config_constants.SUPPRESS_CLOUDWATCH] = False elif arg == "--max-table-size-mb": if value != '' and value is not None: args[config_constants.MAX_TBL_SIZE_MB] = value elif arg == "--min-interleaved-skew": if value != '' and value is not None: args[config_constants.MIN_INTERLEAVED_SKEW] = value elif arg == "--min-interleaved-cnt": if value != '' and value is not None: args[config_constants.MIN_INTERLEAVED_COUNT] = value else: usage("Unsupported Argument " + arg) # Validate that we've got all the args needed if config_constants.DB_NAME not in args: usage("Missing Parameter 'db'") if config_constants.DB_USER not in args: usage("Missing Parameter 'db-user'") if config_constants.DB_PASSWORD not in args: usage("Missing Parameter 'db-pwd'") if config_constants.DB_HOST not in args: usage("Missing Parameter 'db-host'") if config_constants.DB_PORT not in args: usage("Missing Parameter 'db-port'") if config_constants.OUTPUT_FILE in args: sys.stdout = open(args['output_file'], 'w') # invoke the main method of the utility result = analyze_vacuum.run_analyze_vacuum(**args) if result is not None: sys.exit(result) else: sys.exit(0)
def main(argv): supported_args = """db= db-user= db-pwd= db-host= db-port= schema-name= table-name= blacklisted-tables= require-ssl= debug= output-file= slot-count= ignore-errors= query_group= analyze-flag= vacuum-flag= vacuum-parameter= min-unsorted-pct= max-unsorted-pct= deleted-pct= stats-off-pct= predicate-cols= max-table-size-mb= min-interleaved-skew= min-interleaved-cnt=""" # extract the command line arguments try: optlist, remaining = getopt.getopt(argv[1:], "", supported_args.split()) except getopt.GetoptError as err: print(str(err)) usage() args = {} args['db'] = get_env_var('PGDATABASE', None) args['db_user'] = get_env_var('PGUSER', None) args['db_pwd'] = get_env_var('PGPASSWORD', None) args['db_host'] = get_env_var('PGHOST', None) args['db_port'] = get_env_var('PGPORT', 5439) # parse command line arguments for arg, value in optlist: if arg == "--db": if value == '': usage() else: args['db'] = value elif arg == "--db-user": if value == '': usage() else: args['db_user'] = value elif arg == "--db-pwd": if value == '': usage() else: args['db_pwd'] = value elif arg == "--db-host": if value == '': usage() else: args['db_host'] = value elif arg == "--db-port": if value != '' and value is not None: args['db_port'] = int(value) elif arg == "--require-ssl": if value != '' and value is not None: if value.upper() == 'TRUE' or value == '1': args['require_ssl'] = True elif arg == "--schema-name": if value != '' and value is not None: args['schema_name'] = value elif arg == "--table-name": if value != '' and value is not None: args['table_name'] = value elif arg == "--blacklisted-tables": if value != '' and value is not None: args['blacklisted_tables'] = value elif arg == "--debug": if value.upper() == 'TRUE': args['debug'] = True elif arg == "--output-file": # open the supplied file path and bind it to stdout sys.stdout = open(value, 'w') elif arg == "--ignore-errors": if value.upper() == 'TRUE': args['ignore_errors'] = True elif arg == "--slot-count": args['query_slot_count'] = int(value) elif arg == "--query_group": if value != '' and value is not None: args['query_group'] = value elif arg == "--vacuum-flag": if value.upper() == 'FALSE': args['vacuum_flag'] = False else: args['vacuum_flag'] = True elif arg == "--analyze-flag": if value.upper() == 'FALSE': args['analyze_flag'] = False else: args['analyze_flag'] = True elif arg == "--vacuum-parameter": if value.upper() == 'SORT ONLY' or value.upper( ) == 'DELETE ONLY' or value.upper() == 'REINDEX': args['vacuum_parameter'] = value elif arg == "--min-unsorted-pct": if value != '' and value is not None: args['min_unsorted_pct'] = value elif arg == "--max-unsorted-pct": if value != '' and value is not None: args['max_unsorted_pct'] = value elif arg == "--deleted-pct": if value != '' and value is not None: args['deleted_pct'] = value elif arg == "--stats-off-pct": if value != '' and value is not None: args['stats_off_pct'] = value elif arg == "--predicate-cols": if value.upper() == 'TRUE': args['predicate_cols'] = True elif arg == "--max-table-size-mb": if value != '' and value is not None: args['max_table_size_mb'] = value elif arg == "--min-interleaved-skew": if value != '' and value is not None: args['min_interleaved_skew'] = value elif arg == "--min-interleaved-cnt": if value != '' and value is not None: args['min_interleaved_cnt'] = value else: usage("Unsupported Argument " + arg) # Validate that we've got all the args needed if 'db' not in args: usage("Missing Parameter 'db'") if 'db_user' not in args: usage("Missing Parameter 'db-user'") if 'db_pwd' not in args: usage("Missing Parameter 'db-pwd'") if 'db_host' not in args: usage("Missing Parameter 'db-host'") if 'db_port' not in args: usage("Missing Parameter 'db-port'") if 'output_file' in args: sys.stdout = open(args['output_file'], 'w') # invoke the main method of the utility result = analyze_vacuum.run_analyze_vacuum(**args) if result is not None: sys.exit(result) else: sys.exit(0)
def event_handler(event, context): current_region = 'us-east-1' if region_key not in os.environ: print("Warning - using default region %s" % current_region) else: current_region = os.environ[region_key] kms_connection = boto3.client('kms', region_name=current_region) # load the configuration file config_location = config_constants.LOCAL_CONFIG if event is not None and 'ConfigLocation' in event: config_location = event['ConfigLocation'] global debug config = common.get_config(config_location, current_region, debug) if config_constants.DEBUG in config and config[config_constants.DEBUG]: debug = True if debug: print("Configuration File Contents:") print(config) # extract the password use_password = common.get_password(kms_connection, config, debug) # bind the password back into the configuration so we can pass it forward config[config_constants.DB_PASSWORD] = use_password run_utilities = [] if event is not None and "ExecuteUtility" in event: if event["ExecuteUtility"] == config_constants.COLUMN_ENCODING: run_utilities.append(config_constants.COLUMN_ENCODING) elif event["ExecuteUtility"] == config_constants.ANALYZE_VACUUM: run_utilities.append(config_constants.ANALYZE_VACUUM) elif event["ExecuteUtility"] == config_constants.ANALYZE: run_utilities.append(config_constants.ANALYZE) elif event["ExecuteUtility"] == config_constants.VACUUM: run_utilities.append(config_constants.VACUUM) elif event["ExecuteUtility"] == config_constants.MONITORING: run_utilities.append(config_constants.MONITORING) elif event["ExecuteUtility"] == config_constants.TABLE_PERSISTENCE: run_utilities.append(config_constants.TABLE_PERSISTENCE) elif event["ExecuteUtility"] == config_constants.WLM_SCHEDULER: run_utilities.append(config_constants.WLM_SCHEDULER) elif 'utilities' in config: # run each utility, if requested if config_constants.COLUMN_ENCODING in config["utilities"]: run_utilities.append(config_constants.COLUMN_ENCODING) if config_constants.ANALYZE_VACUUM in config["utilities"]: run_utilities.append(config_constants.ANALYZE_VACUUM) if config_constants.ANALYZE in config["utilities"]: run_utilities.append(config_constants.ANALYZE) if config_constants.VACUUM in config["utilities"]: run_utilities.append(config_constants.VACUUM) if config_constants.MONITORING in config["utilities"]: run_utilities.append(config_constants.MONITORING) if config_constants.TABLE_PERSISTENCE in config["utilities"]: run_utilities.append(config_constants.TABLE_PERSISTENCE) if config_constants.WLM_SCHEDULER in config["utilities"]: run_utilities.append(config_constants.WLM_SCHEDULER) else: print("No Utilities configured to run. Exiting!") return results = [] for util in run_utilities: if util == config_constants.COLUMN_ENCODING: print("Running %s" % util) analyze_schema_compression.configure(**config) encoding_result = analyze_schema_compression.run() results.append(encoding_result) elif util == config_constants.ANALYZE_VACUUM: print("Running %s" % util) analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.ANALYZE: print("Running %s" % util) # turn on correct flag config[config_constants.DO_ANALYZE] = True config[config_constants.DO_VACUUM] = False analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.VACUUM: print("Running %s" % util) # turn on correct flag config[config_constants.DO_ANALYZE] = False config[config_constants.DO_VACUUM] = True analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.MONITORING: print("Running %s" % util) redshift_monitoring.monitor_cluster([config, os.environ]) elif util == config_constants.TABLE_PERSISTENCE: print("Running %s" % util) snapshot_system_stats.snapshot([config, os.environ]) elif util == config_constants.WLM_SCHEDULER: print("Running %s" % util) wlm_scheduler.run_scheduler(config) print("Processing Complete") return results
def event_handler(event, context): current_region = 'us-east-1' if region_key not in os.environ: print("Warning - using default region %s" % current_region) else: current_region = os.environ[region_key] kms_connection = boto3.client('kms', region_name=current_region) # load the configuration file config_location = config_constants.LOCAL_CONFIG if event is not None and 'ConfigLocation' in event: config_location = event['ConfigLocation'] global debug config = common.get_config(config_location, current_region, debug) if config_constants.DEBUG in config and config[config_constants.DEBUG]: debug = True if debug: print("Configuration File Contents:") print(config) # extract the password use_password = common.get_password(kms_connection, config, debug) # bind the password back into the configuration so we can pass it forward config[config_constants.DB_PASSWORD] = use_password run_utilities = [] if event is not None and "ExecuteUtility" in event: if event["ExecuteUtility"] == config_constants.COLUMN_ENCODING: run_utilities.append(config_constants.COLUMN_ENCODING) elif event["ExecuteUtility"] == config_constants.ANALYZE_VACUUM: run_utilities.append(config_constants.ANALYZE_VACUUM) elif event["ExecuteUtility"] == config_constants.ANALYZE: run_utilities.append(config_constants.ANALYZE) elif event["ExecuteUtility"] == config_constants.VACUUM: run_utilities.append(config_constants.VACUUM) elif event["ExecuteUtility"] == config_constants.MONITORING: run_utilities.append(config_constants.MONITORING) elif event["ExecuteUtility"] == config_constants.TABLE_PERSISTENCE: run_utilities.append(config_constants.TABLE_PERSISTENCE) elif event["ExecuteUtility"] == config_constants.WLM_SCHEDULER: run_utilities.append(config_constants.WLM_SCHEDULER) elif 'utilities' in config: # run each utility, if requested if config_constants.COLUMN_ENCODING in config["utilities"]: run_utilities.append(config_constants.COLUMN_ENCODING) if config_constants.ANALYZE_VACUUM in config["utilities"]: run_utilities.append(config_constants.ANALYZE_VACUUM) if config_constants.ANALYZE in config["utilities"]: run_utilities.append(config_constants.ANALYZE) if config_constants.VACUUM in config["utilities"]: run_utilities.append(config_constants.VACUUM) if config_constants.MONITORING in config["utilities"]: run_utilities.append(config_constants.MONITORING) if config_constants.TABLE_PERSISTENCE in config["utilities"]: run_utilities.append(config_constants.TABLE_PERSISTENCE) if config_constants.WLM_SCHEDULER in config["utilities"]: run_utilities.append(config_constants.WLM_SCHEDULER) else: print("No Utilities configured to run. Exiting!") return results = [] for util in run_utilities: if util == config_constants.COLUMN_ENCODING: print("Running %s" % util) analyze_schema_compression.configure(**config) encoding_result = analyze_schema_compression.run() results.append(encoding_result) elif util == config_constants.ANALYZE_VACUUM: print("Running %s" % util) analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.ANALYZE: print("Running %s" % util) # turn on correct flag config[config_constants.DO_ANALYZE] = True config[config_constants.DO_VACUUM] = False analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.VACUUM: print("Running %s" % util) # turn on correct flag config[config_constants.DO_ANALYZE] = False config[config_constants.DO_VACUUM] = True analyze_result = analyze_vacuum.run_analyze_vacuum(**config) if analyze_result == 0: results.append("OK") elif util == config_constants.MONITORING: print("Running %s" % util) redshift_monitoring.monitor_cluster([config, os.environ]) elif util == config_constants.TABLE_PERSISTENCE: print("Running %s" % util) snapshot_system_stats.snapshot([config, os.environ]) elif util == config_constants.WLM_SCHEDULER: print("Running %s" % util) wlm_scheduler.run_scheduler(config) print("Processing Complete") return results