def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('user-name', help = 'Name of the user(s) to be deleted.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Require at least one user names if not len(args.user_name): printError("Error, you need to provide at least one user name.") return 42 # Read creds credentials = read_creds(args.profile[0]) if not credentials['AccessKeyId']: return 42 # Connect to IAM APIs iam_client = connect_service('iam', credentials) if not iam_client: return 42 # Delete users for user in args.user_name: delete_user(iam_client, user)
def setup(self): configPrintException(True) self.test_dir = os.path.dirname(os.path.realpath(__file__)) self.test_ruleset_001 = os.path.join(self.test_dir, 'data/test-ruleset.json') self.test_ruleset_002 = os.path.join( self.test_dir, 'data/test-ruleset-absolute-path.json')
def main(): # Parse arguments parser = RulesArgumentParser() args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Load ruleset ruleset = Ruleset(filename=args.base_ruleset, name=args.ruleset_name, load_rules=False, rules_dir=args.rules_dir) # Generate the HTML generator ruleset_generator = RulesetGenerator(args.ruleset_name, args.generator_dir) ruleset.ruleset_generator_metadata = Scout2Config('default', None, None, [], []).metadata ruleset_generator_path = ruleset_generator.save(ruleset, args.force_write, args.debug) # Open the HTML ruleset generator in a browser printInfo('Starting the HTML ruleset generator...') url = 'file://%s' % os.path.abspath(ruleset_generator_path) webbrowser.open(url, new=2)
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.parser.add_argument('--user-name', dest='user_name', default=None, nargs='+', required='True', help='Name of the user(s) to be deleted.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Read creds credentials = read_creds(args.profile[0]) if not credentials['AccessKeyId']: return 42 # Connect to IAM APIs iam_client = connect_service('iam', credentials) # Delete users for user in args.user_name: delete_user(iam_client, user)
def setup(self): configPrintException(True) self.creds = read_creds_from_environment_variables() if self.creds['AccessKeyId'] == None: self.creds = read_creds('travislike') self.api_client = connect_service('iam', self.creds) self.python = re.sub(r'\W+', '', sys.version) self.cleanup = {'groups': [], 'users': []}
def setup(self): configPrintException(True) self.tmp_aws_dir = '%s-opineltest' % aws_dir if os.path.isdir(aws_dir): os.rename(aws_dir, self.tmp_aws_dir) os.mkdir(aws_dir) shutil.copyfile('tests/data/config', os.path.join(aws_dir, 'config')) shutil.copyfile('tests/data/credentials', os.path.join(aws_dir, 'credentials'))
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument( 'user-name', help_string= 'Your username (automatically fetched using iam:GetUser if not provided).' ) args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Arguments profile_name = args.profile[0] user_name = args.user_name[0] # Search for AWS credentials credentials = read_creds(profile_name) if not credentials['AccessKeyId']: return 42 # Connect to IAM iam_client = connect_service('iam', credentials) if not iam_client: printError('Error: failed to create IAM API client.') return 42 # Set the user name if not user_name: try: printInfo('Searching for username...') user_name = iam_client.get_user()['User']['UserName'] if not user_name: printInfo('Error: could not find user name to enable MFA for.') return 42 except Exception as e: configPrintException(e) # Create and activate the MFA device credentials['SerialNumber'] = enable_mfa(iam_client, user_name) # Update the credentials file write_creds_to_aws_credentials_file(profile_name, credentials) sample_command = 'awsrecipes_init_sts_session.py %s' % ( ('--profile %s' % profile_name) if profile_name != 'default' else '') printInfo('Your credentials file has been updated.\n' \ 'You may now initiate STS sessions to access the AWS APIs with the following command:\n' \ '\n %s\n' % sample_command)
def setup(self): configPrintException(True) self.creds = read_creds_from_environment_variables() if self.creds['AccessKeyId'] == None: self.creds = read_creds('travislike') self.api_client = connect_service('cloudformation', self.creds, 'us-east-1') self.python = re.sub(r'\W+', '', sys.version) self.cleanup = {'stacks': [], 'stacksets': []}
def setup(self): configPrintException(True) self.creds = read_creds_from_environment_variables() if self.creds['AccessKeyId'] == None: self.creds = read_creds('travislike') self.org_creds = assume_role( 'OpinelUnitTest', self.creds, 'arn:aws:iam::990492604467:role/OpinelUnitTest', 'opinelunittesting') self.badapi_client = connect_service('organizations', self.creds, 'us-east-1') self.api_client = connect_service('organizations', self.org_creds, 'us-east-1')
def main(): # Parse arguments parser = OpinelArgumentParser(os.path.basename(__file__)) parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('common-groups', default=[], nargs='+', help='List of groups each IAM user should belong to.') parser.add_argument( 'category-groups', default=[], nargs='+', help='List of category groups; each IAM user must belong to one.') parser.add_argument( 'category-regex', default=[], nargs='+', help='List of regex enabling auto-assigment of category groups.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Get profile name profile_name = args.profile[0] # Search for AWS credentials credentials = read_creds(profile_name) if not credentials['AccessKeyId']: return 42 # Connect to IAM iam_client = connect_service('iam', credentials) if not iam_client: return 42 # Create groups for group in args.category_groups + args.common_groups: try: printInfo('Creating group %s...' % group) iam_client.create_group(GroupName=group) except Exception as e: if e.response['Error']['Code'] != 'EntityAlreadyExists': printException(e)
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('mfa-serial') parser.add_argument('mfa-code') parser.parser.add_argument('--role-session-name', dest='role_session_name', default=None, help='Identifier for the assumed role\'s session.') parser.parser.add_argument('--role-name', dest='role_name', default=None, help='Identifier for the assumed role.') parser.parser.add_argument('--role-arn', dest='role_arn', default=None, help='ARN of the assumed role.') parser.parser.add_argument('--external-id', dest='external_id', default=None, help='External ID to use when assuming the role.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Read creds automatically prompts for MFA code and assumes a role if role is already configured try: credentials = read_creds(args.profile[0], mfa_code = args.mfa_code, mfa_serial_arg = args.mfa_serial) except Exception as e: printException(e) return 42 # If the role's ARN was provided... if args.role_arn: if not args.role_name: printError('Error: you must specify a name for this role.') return 42 role_session_name = args.role_session_name if args.role_session_name else 'aws-recipes-%s' % str(datetime.datetime.utcnow()).replace(' ', '_').replace(':','-') assume_role(args.role_name, credentials, args.role_arn, role_session_name)
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('mfa-serial') parser.add_argument('mfa-code') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Read creds automatically prompts for MFA code and initiates a session if expired credentials = read_creds(args.profile[0], mfa_code = args.mfa_code, mfa_serial_arg = args.mfa_serial, force_init = True) if not credentials['AccessKeyId']: return 42
class TestScout2CloudWatchUtilsClass: configPrintException(True) # # Test get_cloudwatch_region in us-east-1 # def test_get_cloudwatch_region(self): # TODO: change to us-east-1 credentials = read_creds('default') service_config = {'regions': {'us-east-1': {}}} get_cloudwatch_region( params={ 'region': 'us-east-1', 'creds': credentials, 'cloudwatch_config': service_config }) # # Test get_cloudwatch_info (multiple thread of get_cloudwatch_region) # 1. in us-east-1 and us-west-1 # 2. in empty region intersection # def test_get_cloudwatch_info(self): credentials = read_creds('default') service_config = { 'regions': { 'us-east-1': {}, 'us-west-1': {} } } # , 'cn-north-1': {}}} get_cloudwatch_info(credentials, service_config, ['us-east-1', 'us-west-1'], 'aws') get_cloudwatch_info(credentials, service_config, ['us-east-1', 'us-west-1'], 'aws-us-gov') # get_cloudwatch_info(credentials, service_config, ['us-gov-west-1'], 'aws-us-gov') # # Smoke tests for status display functions # def test_cloudwatch_status_init(self): cloudwatch_status_init() def test_cloudwatch_status(self): cloudwatch_status(True) cloudwatch_status(False) cloudwatch_status() def test_formatted_status(self): formatted_status(1, 42, True) formatted_status(42, 1, False)
def main(): # Parse arguments parser = RulesArgumentParser() args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Load ruleset ruleset = Ruleset(filename=args.base_ruleset, name=args.ruleset_name, rules_dir=args.rules_dir, ruleset_generator=True) # Generate the HTML generator ruleset_generator = RulesetGenerator(args.ruleset_name, args.generator_dir) #FIXME this is broken in Scout Suite # Create a cloud provider object cloud_provider = get_provider(provider='aws', profile='default') ruleset.ruleset_generator_metadata = Scout2Config('default', None, None, [], []).metadata ruleset_generator_path = ruleset_generator.save(ruleset, args.force_write, args.debug) # Open the HTML ruleset generator in a browser if not args.no_browser: printInfo('Starting the HTML ruleset generator...') url = 'file://%s' % os.path.abspath(ruleset_generator_path) webbrowser.open(url, new=2)
def main(args): setup_logging(args.debug) if args.version: print(aws_inventory.__version__) return api_model = build_api_model() if args.list_svcs: print('\n'.join(sorted(filter_services(api_model)))) return # configure the debug level for opinel configPrintException(args.debug) # validate services against API mode # available_services = api_model.keys() if args.services: invalid_included_services = [ svc for svc in args.services if svc not in available_services ] if invalid_included_services: raise EnvironmentError('Invalid service(s) specified: {}'.format( ', '.join(invalid_included_services))) if args.excluded_services: invalid_excluded_services = [ svc for svc in args.excluded_services if svc not in available_services ] if invalid_excluded_services: raise EnvironmentError('Invalid service(s) to exclude: {}'.format( ', '.join(invalid_excluded_services))) # validate regions against API model # if args.regions: available_regions = set() for svc in available_services: available_regions.update(api_model[svc]['regions']) invalid_regions = [ region for region in args.regions if region not in available_regions ] if invalid_regions: raise EnvironmentError('Invalid region(s) specified: {}'.format( ', '.join(invalid_regions))) # create the list of services to analyze services = filter_services(api_model, frozenset(args.services), frozenset(args.excluded_services)) if not services: raise EnvironmentError('List of AWS services to be analyzed is empty.') LOGGER.debug('%d AWS service(s) to inspect: %s.', len(services), ', '.join(services)) op_blacklist_parser = aws_inventory.blacklist.OpBlacklistParser( args.op_blacklist, api_model) service_descriptors = filter_operations(api_model, op_blacklist_parser, args.regions, services) if not service_descriptors: raise EnvironmentError( 'No operations to invoke for specifed AWS services and regions.') ops_count = 0 for svc_name in service_descriptors: ops_count += (len(service_descriptors[svc_name]['ops']) * len(service_descriptors[svc_name]['regions'])) if args.list_operations: print('[{}]\n{}\n'.format( svc_name, '\n'.join(service_descriptors[svc_name]['ops']) or '# NONE')) profiles = [args.profile] if args.regex: # find unique profiles that match the regex profiles = list( set( map(lambda profile: profile.split()[-1], get_profiles_from_aws_credentials_file()))) profiles = list( filter(lambda profile: re.search(args.profile, profile), profiles)) tqdm.tqdm.monitor_interval = 0 if args.list_operations: print('Total operations to invoke: {}'.format(ops_count * len(profiles))) else: LOGGER.debug('Total operations to invoke: %d.', ops_count * len(profiles)) for profile in profiles: args.profile = profile gui_data_file = data_file_name( args.gui_file_template, profile) if 'gui' in args.output else None response_data_file = data_file_name( args.response_file_template, profile) if 'response' in args.output else None exception_data_file = data_file_name( args.exception_file_template, profile) if 'exception' in args.output else None aws_inventory.invoker.ApiInvoker( args, service_descriptors, ops_count, gui_data_file, response_data_file, exception_data_file).probe_services()
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('csv-credentials') parser.add_argument('mfa-serial') parser.add_argument('mfa-code') parser.parser.add_argument('--role-arn', dest='role_arn', default=None, help='ARN of the assumed role.') parser.parser.add_argument('--external-id', dest='external_id', default=None, help='External ID to use when assuming the role.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Arguments profile_name = args.profile[0] if args.csv_credentials: # Read credentials from a CSV file credentials = {} credentials['AccessKeyId'], credentials['SecretAccessKey'], credentials['SerialNumber'] = read_creds_from_csv(args.csv_credentials) if not credentials['AccessKeyId'] or not credentials['SecretAccessKey']: printError('Failed to read credentials from %s' % args.csv_credentials) return 42 use_found_credentials = True else: # Check for migration from existing profile to no-mfa profile use_found_credentials = False credentials = read_creds_from_aws_credentials_file(profile_name) if 'AccessKeyId' in credentials and credentials['AccessKeyId'] != None and credentials['SecretAccessKey'] != None and credentials['SerialNumber'] == None and credentials['SessionToken'] == None: if prompt_4_yes_no('Found long-lived credentials for the profile \'%s\'. Do you want to use those when configuring MFA' % profile_name): use_found_credentials = True iam_client = connect_service('iam', credentials) try: printInfo('Trying to read the MFA serial number associated with this IAM user...') user_name = iam_client.get_user()['User']['UserName'] mfa_devices = iam_client.list_mfa_devices(UserName = user_name)['MFADevices'] credentials['SerialNumber'] = mfa_devices[0]['SerialNumber'] except Exception as e: printException(e) pass if not use_found_credentials: # Get values credentials['AccessKeyId'] = prompt_4_value('AWS Access Key ID: ', no_confirm = True) credentials['SecretAccessKey'] = prompt_4_value('AWS Secret Access Key: ', no_confirm = True) if 'SerialNumber' not in credentials or not credentials['SerialNumber']: credentials['SerialNumber'] = prompt_4_mfa_serial() # Check for overwrite while True: c = read_creds_from_aws_credentials_file(profile_name) if 'AccessKeyId' in c and c['AccessKeyId']: if not prompt_4_yes_no('The profile \'%s\' already exists. Do you want to overwrite the existing values' % profile_name): if not prompt_4_yes_no('Do you want to create a new profile with these credentials'): printError('Configuration aborted.') return profile_name = prompt_4_value('Profile name: ') else: break else: break # Write values to credentials file write_creds_to_aws_credentials_file(profile_name, credentials) # Delete CSV file? if args.csv_credentials and prompt_4_yes_no('Do you want to delete the CSV file that contains your long-lived credentials?'): os.remove(args.csv_credentials)
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('managed', dest='is_managed', default=False, action='store_true', help='Create a managed policy.') parser.add_argument( 'type', default=[None], nargs='+', choices=['group', 'managed', 'role', 'user'], help='Type of target that the policy will apply or be attached to.') parser.add_argument( 'targets', default=[], nargs='+', help= 'Name of the IAM entity the policy will be added to (required for inline policies).' ) parser.add_argument( 'templates', default=[], nargs='+', help='Path to the template IAM policies that will be created.') parser.add_argument('save', dest='save_locally', default=False, action='store_true', help='Generates the policies and store them locally.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Arguments profile_name = args.profile[0] target_type = args.type[0] if len(args.templates) == 0: printError( 'Error: you must specify the path the template IAM policies.') return 42 if not args.is_managed and target_type == None: printError( 'Error: you must either create a managed policy or specify the type of IAM entity the policy will be attached to.' ) return 42 if not args.is_managed and target_type == None and len(args.targets) < 1: printError( 'Error: you must provide the name of at least one IAM %s you will attach this inline policy to.' % target_type) return 42 # Read creds credentials = read_creds(args.profile[0]) if not credentials['AccessKeyId']: return 42 # Connect to IAM APIs iam_client = connect_service('iam', credentials) if not iam_client: return 42 # Get AWS account ID aws_account_id = get_aws_account_id(credentials) # Create the policies for template in args.templates: if not os.path.isfile(template): printError('Error: file \'%s\' does not exist.' % template) continue with open(template, 'rt') as f: policy = f.read() policy = re_aws_account_id.sub(aws_account_id, policy) policy_name = os.path.basename(template).split('.')[0] if not args.is_managed: callback = getattr(iam_client, 'put_' + target_type + '_policy') params = {} params['PolicyName'] = policy_name params['PolicyDocument'] = policy for target in args.targets: params[target_type.title() + 'Name'] = target try: printInfo( 'Creating policy \'%s\' for the \'%s\' IAM %s...' % (policy_name, target, target_type)) callback(**params) except Exception as e: printException(e) pass else: params = {} params['PolicyDocument'] = policy params['PolicyName'] = policy_name description = '' # Search for a description file descriptions_dir = os.path.join(os.path.dirname(template), 'descriptions') if os.path.exists(descriptions_dir): description_file = os.path.join( descriptions_dir, os.path.basename(template).replace('.json', '.txt')) if os.path.isfile(description_file): with open(description_file, 'rt') as f: params['Description'] = f.read() elif prompt_4_yes_no( 'Do you want to add a description to the \'%s\' policy' % policy_name): params['Description'] = prompt_4_value( 'Enter the policy description:') params['Description'] = params['Description'].strip() printInfo('Creating policy \'%s\'...' % (policy_name)) new_policy = iam_client.create_policy(**params) if len(args.targets): callback = getattr(iam_client, 'attach_' + target_type + '_policy') for target in args.targets: printInfo('Attaching policy to the \'%s\' IAM %s...' % (target, target_type)) params = {} params['PolicyArn'] = new_policy['Policy']['Arn'] params[target_type.title() + 'Name'] = target callback(**params) if args.save_locally: with open('%s-%s.json' % (policy_name, profile_name), 'wt') as f: f.write(policy) f.close()
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('user-name', help='Name of the IAM user(s).') parser.parser.add_argument('--all-users', dest='all_users', default=False, action='store_true', help='Go through all IAM users') parser.parser.add_argument( '--arn', dest='arn', default=[], nargs='+', help='ARN of the target group(s), role(s), or user(s)') parser.parser.add_argument('--group-name', dest='group_name', default=[], nargs='+', help='Name of the IAM group(s)') parser.parser.add_argument('--all-groups', dest='all_groups', default=False, action='store_true', help='Go through all IAM groups') parser.parser.add_argument('--role-name', dest='role_name', default=[], nargs='+', help='Name of the IAM role(s)') parser.parser.add_argument('--all-roles', dest='all_roles', default=False, action='store_true', help='Go through all IAM roles') parser.parser.add_argument('--policy-arn', dest='policy_arn', default=[], nargs='+', help='ARN of the IAM policy/ies') parser.parser.add_argument('--all', dest='all', default=False, action='store_true', help='Go through all IAM resources') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Arguments profile_name = args.profile[0] # Search for AWS credentials credentials = read_creds(profile_name) if not credentials['AccessKeyId']: return 42 # Connect to IAM iam_client = connect_service('iam', credentials) if not iam_client: return 42 # Normalize targets targets = [] for arn in args.arn: arn_match = re_arn.match(arn) if arn_match: resource = arn_match.groups()[4].split('/') targets.append((resource[0], resource[-1])) for group_name in args.group_name: if group_name: targets.append(('group', group_name)) for role_name in args.role_name: if role_name: targets.append(('role', role_name)) for user_name in args.user_name: if user_name: targets.append(('user', user_name)) if args.all or args.all_groups: printInfo('Fetching all IAM groups...') for group in handle_truncated_response(iam_client.list_groups, {}, ['Groups'])['Groups']: targets.append(('group', group['GroupName'])) if args.all or args.all_roles: printInfo('Fetching all IAM roles...') for role in handle_truncated_response(iam_client.list_roles, {}, ['Roles'])['Roles']: targets.append(('role', role['RoleName'])) if args.all or args.all_users: printInfo('Fetching all IAM users...') for user in handle_truncated_response(iam_client.list_users, {}, ['Users'])['Users']: targets.append(('user', user['UserName'])) # Get all policies that apply to the targets and aggregate them into a single file printInfo('Fetching all inline and managed policies in use...') managed_policies = {} for resource_type, resource_name in targets: policy_documents = get_policies(iam_client, managed_policies, resource_type, resource_name) write_permissions(merge_policies(policy_documents), resource_type, resource_name) # Get requested managed policies for policy_arn in args.policy_arn: policy_documents = [ get_managed_policy_document(iam_client, policy_arn, managed_policies) ] write_permissions(merge_policies(policy_documents), 'policy', policy_arn)
def setup(self): configPrintException(True)
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('regions') parser.add_argument('partition-name') parser.parser.add_argument('--filters', dest='filters', default=None, help='') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Get profile name profile_name = args.profile[0] # Build list of region regions = build_region_list('ec2', args.regions, args.partition_name) printInfo(str(regions)) # Build filters filters = json.loads(args.filters) if args.filters else None # Search for AWS credentials credentials = read_creds(profile_name) if not credentials['AccessKeyId']: return 42 # List all EC2 instances instances = [] for region in regions: printInfo('Fetching instances in %s...' % region) ec2_client = connect_service('ec2', credentials, region_name=region) args = {'Filters': filters} if filters else {} reservations = handle_truncated_response( ec2_client.describe_instances, args, ['Reservations'])['Reservations'] for r in reservations: instances += r['Instances'] printInfo(' Found %d instances' % len(instances)) # Build list of private and public IPs prvips = {} pubips = {} for i in instances: security_groups = i['SecurityGroups'] for eni in i['NetworkInterfaces']: for prvip in eni['PrivateIpAddresses']: prvips[prvip['PrivateIpAddress']] = { 'security_groups': security_groups } if 'Association' in prvip: pubips[prvip['Association']['PublicIp']] = { 'security_groups': security_groups } # Create target files with open('targets-%s-prv.txt' % profile_name, 'wt') as f: for prvip in prvips: f.write('%s\n' % prvip) with open('targets-%s-pub.txt' % profile_name, 'wt') as f: for pubip in pubips: f.write('%s\n' % pubip)
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('force') parser.add_argument('dry-run') parser.add_argument('regions') parser.add_argument('partition-name') parser.parser.add_argument('--interactive', dest='interactive', default=False, action='store_true', help='Interactive prompt to manually enter CIDRs.') parser.parser.add_argument('--csv-ip-ranges', dest='csv_ip_ranges', default=[], nargs='+', help='CSV file(s) containing CIDRs information.') parser.parser.add_argument('--skip-first-line', dest='skip_first_line', default=False, action='store_true', help='Skip first line when parsing CSV file.') parser.parser.add_argument('--attributes', dest='attributes', default=[], nargs='+', help='Name of the attributes to enter for each CIDR.') parser.parser.add_argument('--mappings', dest='mappings', default=[], nargs='+', help='Column number matching attributes when headers differ.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Initialize the list of regions to work with regions = build_region_list('ec2', args.regions, args.partition_name) # For each profile/environment... for profile_name in args.profile: # Interactive mode if args.interactive: # Initalize prefixes attributes = args.attributes filename = 'ip-ranges-%s.json' % profile_name if os.path.isfile(filename): printInfo('Loading existing IP ranges from %s' % filename) prefixes = read_ip_ranges(filename) # Initialize attributes from existing values if attributes == []: for prefix in prefixes: for key in prefix: if key not in attributes: attributes.append(key) else: prefixes = [] # IP prefix does not need to be specified as an attribute attributes = [a for a in attributes if a != 'ip_prefix'] # Prompt for new entries while prompt_4_yes_no('Add a new IP prefix to the ip ranges'): ip_prefix = prompt_4_value('Enter the new IP prefix:') obj = {} for a in attributes: obj[a] = prompt_4_value('Enter the \'%s\' value:' % a) prefixes.append(new_prefix(ip_prefix, obj)) # Support loading from CSV file elif len(args.csv_ip_ranges) > 0: # Initalize prefixes prefixes = [] # Load CSV file contents for filename in args.csv_ip_ranges: with open(filename, 'rt') as f: csv_contents = f.readlines() # Initialize mappings attributes = args.attributes mappings = {} if attributes == []: # Follow structure of first line headers = csv_contents.pop(0).strip().split(',') for index, attribute in enumerate(headers): mappings[attribute] = index elif attributes and args.mappings == []: # Follow structure of first line but only map a subset of fields headers = csv_contents.pop(0).strip().split(',') attributes.append('ip_prefix') for attribute in set(attributes): mappings[attribute] = headers.index(attribute) else: # Indices of columns are provided as an argument for index, attribute in enumerate(attributes): mappings[attribute] = int(args.mappings[index]) if args.skip_first_line: csv_contents.pop(0) # For each line... for line in csv_contents: ip_prefix = {} values = line.strip().split(',') if len(values) < len(mappings): continue for attribute in mappings: ip_prefix[attribute] = values[mappings[attribute]] if 'ip_prefix' in mappings and 'mask' in mappings: ip = ip_prefix.pop('ip_prefix') mask = ip_prefix.pop('mask') ip_prefix['ip_prefix'] = '%s/%s' % (ip, mask.replace('/','')) prefixes.append(ip_prefix) # AWS mode else: # Initialize IP addresses printInfo('Fetching public IP information for the \'%s\' environment...' % profile_name) ip_addresses = {} # Search for AWS credentials credentials = read_creds(profile_name) if not credentials['AccessKeyId']: return 42 # For each region... for region in regions: # Connect to EC2 ec2_client = connect_service('ec2', credentials, region) if not ec2_client: continue # Get public IP addresses associated with EC2 instances printInfo('...in %s: EC2 instances' % region) reservations = handle_truncated_response(ec2_client.describe_instances, {}, ['Reservations']) for reservation in reservations['Reservations']: for i in reservation['Instances']: if 'PublicIpAddress' in i: ip_addresses[i['PublicIpAddress']] = new_ip_info(region, i['InstanceId'], False) get_name(i, ip_addresses[i['PublicIpAddress']], 'InstanceId') if 'NetworkInterfaces' in i: for eni in i['NetworkInterfaces']: if 'Association' in eni: ip_addresses[eni['Association']['PublicIp']] = new_ip_info(region, i['InstanceId'], False) # At that point, we don't know whether it's an EIP or not... get_name(i, ip_addresses[eni['Association']['PublicIp']], 'InstanceId') # Get all EIPs (to handle unassigned cases) printInfo('...in %s: Elastic IP addresses' % region) eips = handle_truncated_response(ec2_client.describe_addresses, {}, ['Addresses']) for eip in eips['Addresses']: instance_id = eip['InstanceId'] if 'InstanceId' in eip else None # EC2-Classic non associated EIPs have an empty string for instance ID (instead of lacking the attribute in VPC) if instance_id == '': instance_id = None ip_addresses[eip['PublicIp']] = new_ip_info(region, instance_id, True) ip_addresses[eip['PublicIp']]['name'] = instance_id # Format prefixes = [] for ip in ip_addresses: prefixes.append(new_prefix(ip, ip_addresses[ip])) # Generate an ip-ranges-<profile>.json file save_ip_ranges(profile_name, prefixes, args.force_write, args.debug)
def main(): # Parse arguments parser = ListallArgumentParser() args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Support multiple environments for profile_name in args.profile: # Load the config try: report = Scout2Report(profile_name, args.report_dir, args.timestamp) aws_config = report.jsrw.load_from_file(AWSCONFIG) services = aws_config['service_list'] except Exception as e: printException(e) printError('Error, failed to load the configuration for profile %s' % profile_name) continue # Create a ruleset with only whatever rules were specified... if args.config: rule_filename = args.config ruleset = TmpRuleset(rule_dirs = [os.getcwd()], rule_filename = args.config, rule_args = args.config_args) elif len(args.path) > 0: # Create a local tmp rule rule_dict = {'description': 'artifact'} rule_dict['path'] = args.path[0] rule_dict['conditions'] = [] rule_filename = 'listall-artifact.json' with open(os.path.join(os.getcwd(), rule_filename), 'wt') as f: f.write(json.dumps(rule_dict)) ruleset = TmpRuleset(rule_dirs = [os.getcwd()], rule_filename = rule_filename, rule_args = []) else: printError('Error, you must provide either a rule configuration file or the path to the resources targeted.') continue # Process the rule pe = ProcessingEngine(ruleset) pe.run(aws_config, skip_dashboard = True) # Retrieve items rule = ruleset.rules[rule_filename][0] rule_service = rule.service.lower() rule_key = rule.key rule_type = rule.rule_type resources = aws_config['services'][rule_service][rule_type][rule_key]['items'] # Set the keys to output if len(args.keys): # 1. Explicitly provided on the CLI rule.keys = args.keys elif len(args.keys_file): # 2. Explicitly provided files that contain the list of keys rule.keys = [] for filename in args.keys_file: with open(filename, 'rt') as f: rule.keys += json.load(f)['keys'] else: try: # 3. Load default set of keys based on path target_path = rule.display_path if hasattr(rule, 'display_path') else rule.path listall_configs_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'output/data/listall-configs') target_file = os.path.join(listall_configs_dir, '%s.json' % target_path) if os.path.isfile(target_file): with open(target_file, 'rt') as f: rule.keys = json.load(f)['keys'] except: # 4. Print the object name rule.keys = ['name'] # Prepare the output format (lines, template) = format_listall_output(args.format_file[0], None, args.format, rule) # Print the output printInfo(generate_listall_output(lines, resources, aws_config, template, []))
def main(): # Parse arguments parser = Scout2ArgumentParser() args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Set the profile name profile_name = args.profile[0] # Search for AWS credentials if not args.fetch_local: credentials = read_creds(args.profile[0], args.csv_credentials, args.mfa_serial, args.mfa_code) if credentials['AccessKeyId'] is None: return 42 # Create a new Scout2 config report = Scout2Report(profile_name, args.report_dir, args.timestamp) aws_config = Scout2Config(profile_name, args.report_dir, args.timestamp, args.services, args.skipped_services) if not args.fetch_local: # Fetch data from AWS APIs if not running a local analysis try: aws_config.fetch(credentials, regions=args.regions, partition_name=args.partition_name) except KeyboardInterrupt: printInfo('\nCancelled by user') return 130 aws_config = report.jsrw.to_dict(aws_config) # Update means we reload the whole config and overwrite part of it if args.update == True: new_aws_config = copy.deepcopy(aws_config) aws_config = report.jsrw.load_from_file(AWSCONFIG) for service in new_aws_config['service_list']: # Per service only for now, may add per region & per VPC later... aws_config['services'][service] = new_aws_config['services'][service] else: # Reload to flatten everything into a python dictionary aws_config = report.jsrw.load_from_file(AWSCONFIG) # Pre processing preprocessing(aws_config, args.ip_ranges, args.ip_ranges_name_key) # Analyze config ruleset = Ruleset(profile_name, filename = args.ruleset, ip_ranges = args.ip_ranges) ruleset.analyze(aws_config) # Create display filters filters = Ruleset(filename = 'filters.json', rule_type = 'filters') filters.analyze(aws_config) # Handle exceptions process_exceptions(aws_config, args.exceptions[0]) # Finalize postprocessing(aws_config, report.current_time, ruleset) # Save config and create HTML report html_report_path = report.save(aws_config, {}, args.force_write, args.debug) # Open the report by default if not args.no_browser: printInfo('Opening the HTML report...') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2)
def setup(self): configPrintException(True) self.test_dir = os.path.dirname(os.path.realpath(__file__)) self.test_ruleset_001 = os.path.join(self.test_dir, 'data/test-ruleset.json') self.test_ruleset_002 = os.path.join(self.test_dir, 'data/test-ruleset-absolute-path.json')
def main(args=None): """ Main method that runs a scan :return: """ if not args: parser = ScoutSuiteArgumentParser() args = parser.parse_args() # Get the dictionnary to get None instead of a crash args = args.__dict__ # Configure the debug level configPrintException(args.get('debug')) # Create a cloud provider object cloud_provider = get_provider( provider=args.get('provider'), profile=args.get('profile'), project_id=args.get('project_id'), folder_id=args.get('folder_id'), organization_id=args.get('organization_id'), all_projects=args.get('all_projects'), report_dir=args.get('report_dir'), timestamp=args.get('timestamp'), services=args.get('services'), skipped_services=args.get('skipped_services'), thread_config=args.get('thread_config'), ) report_file_name = generate_report_name(cloud_provider.provider_code, args) # TODO move this to after authentication, so that the report can be more specific to what's being scanned. # For example if scanning with a GCP service account, the SA email can only be known after authenticating... # Create a new report report = Scout2Report(args.get('provider'), report_file_name, args.get('report_dir'), args.get('timestamp')) # Complete run, including pulling data from provider if not args.get('fetch_local'): # Authenticate to the cloud provider authenticated = cloud_provider.authenticate( profile=args.get('profile'), csv_credentials=args.get('csv_credentials'), mfa_serial=args.get('mfa_serial'), mfa_code=args.get('mfa_code'), user_account=args.get('user_account'), service_account=args.get('service_account'), cli=args.get('cli'), msi=args.get('msi'), service_principal=args.get('service_principal'), file_auth=args.get('file_auth'), tenant_id=args.get('tenant_id'), subscription_id=args.get('subscription_id'), client_id=args.get('client_id'), client_secret=args.get('client_secret'), username=args.get('username'), password=args.get('password')) if not authenticated: return 42 # Fetch data from provider APIs try: cloud_provider.fetch(regions=args.get('regions')) except KeyboardInterrupt: printInfo('\nCancelled by user') return 130 # Update means we reload the whole config and overwrite part of it if args.get('update'): current_run_services = copy.deepcopy(cloud_provider.services) last_run_dict = report.jsrw.load_from_file(AWSCONFIG) cloud_provider.services = last_run_dict['services'] for service in cloud_provider.service_list: cloud_provider.services[service] = current_run_services[ service] # Partial run, using pre-pulled data else: # Reload to flatten everything into a python dictionary last_run_dict = report.jsrw.load_from_file(AWSCONFIG) for key in last_run_dict: setattr(cloud_provider, key, last_run_dict[key]) # Pre processing cloud_provider.preprocessing(args.get('ip_ranges'), args.get('ip_ranges_name_key')) # Analyze config finding_rules = Ruleset(environment_name=args.get('profile'), cloud_provider=args.get('provider'), filename=args.get('ruleset'), ip_ranges=args.get('ip_ranges'), aws_account_id=cloud_provider.aws_account_id) processing_engine = ProcessingEngine(finding_rules) processing_engine.run(cloud_provider) # Create display filters filter_rules = Ruleset(cloud_provider=args.get('provider'), filename='filters.json', rule_type='filters', aws_account_id=cloud_provider.aws_account_id) processing_engine = ProcessingEngine(filter_rules) processing_engine.run(cloud_provider) # Handle exceptions try: exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: printDebug( 'Warning, failed to load exceptions. The file may not exist or may have an invalid format.' ) exceptions = {} # Finalize cloud_provider.postprocessing(report.current_time, finding_rules) # TODO this is AWS-specific - move to postprocessing? # Get organization data if it exists try: profile = AWSProfiles.get(args.get('profile'))[0] if 'source_profile' in profile.attributes: organization_info_file = os.path.join( os.path.expanduser('~/.aws/recipes/%s/organization.json' % profile.attributes['source_profile'])) if os.path.isfile(organization_info_file): with open(organization_info_file, 'rt') as f: org = {} accounts = json.load(f) for account in accounts: account_id = account.pop('Id') org[account_id] = account setattr(cloud_provider, 'organization', org) except Exception as e: pass # Save config and create HTML report html_report_path = report.save(cloud_provider, exceptions, args.get('force_write'), args.get('debug')) # Open the report by default if not args.get('no_browser'): printInfo('Opening the HTML report...') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2) return 0
def main(): # Parse arguments parser = Scout2ArgumentParser() args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Set the profile name profile_name = args.profile[0] # Search for AWS credentials if not args.fetch_local: credentials = read_creds(args.profile[0], args.csv_credentials, args.mfa_serial, args.mfa_code) if credentials['AccessKeyId'] is None: return 42 # Create a new Scout2 config report = Scout2Report(profile_name, args.report_dir, args.timestamp) aws_config = Scout2Config(profile_name, args.report_dir, args.timestamp, args.services, args.skipped_services, args.thread_config) if not args.fetch_local: # Fetch data from AWS APIs if not running a local analysis try: aws_config.fetch(credentials, regions=args.regions, partition_name=get_partition_name(credentials)) except KeyboardInterrupt: printInfo('\nCancelled by user') return 130 aws_config = report.jsrw.to_dict(aws_config) # Set the account ID aws_config['aws_account_id'] = get_aws_account_id(credentials) # Update means we reload the whole config and overwrite part of it if args.update == True: new_aws_config = copy.deepcopy(aws_config) aws_config = report.jsrw.load_from_file(AWSCONFIG) for service in new_aws_config['service_list']: # Per service only for now, may add per region & per VPC later... aws_config['services'][service] = new_aws_config['services'][ service] # Update the metadata too aws_config['metadata'] = Scout2Config('default', None, None, [], []).metadata else: # Reload to flatten everything into a python dictionary aws_config = report.jsrw.load_from_file(AWSCONFIG) # Pre processing preprocessing(aws_config, args.ip_ranges, args.ip_ranges_name_key) # Analyze config finding_rules = Ruleset(profile_name, filename=args.ruleset, ip_ranges=args.ip_ranges, aws_account_id=aws_config['aws_account_id']) pe = ProcessingEngine(finding_rules) pe.run(aws_config) # Create display filters filter_rules = Ruleset(filename='filters.json', rule_type='filters', aws_account_id=aws_config['aws_account_id']) pe = ProcessingEngine(filter_rules) pe.run(aws_config) # Handle exceptions try: exceptions = RuleExceptions(profile_name, args.exceptions[0]) exceptions.process(aws_config) exceptions = exceptions.exceptions except Exception as e: printDebug( 'Warning, failed to load exceptions. The file may not exist or may have an invalid format.' ) exceptions = {} # Finalize postprocessing(aws_config, report.current_time, finding_rules) # Get organization data if it exists try: profile = AWSProfiles.get(profile_name)[0] if 'source_profile' in profile.attributes: organization_info_file = os.path.join( os.path.expanduser('~/.aws/recipes/%s/organization.json' % profile.attributes['source_profile'])) if os.path.isfile(organization_info_file): with open(organization_info_file, 'rt') as f: org = {} accounts = json.load(f) for account in accounts: account_id = account.pop('Id') org[account_id] = account aws_config['organization'] = org except: pass if args.json: printInfo('Writing to results.json') fp = open('results.json', 'w') json.dump(aws_config, fp, default=json_helper) fp.close() sys.exit() # Save config and create HTML report html_report_path = report.save(aws_config, exceptions, args.force_write, args.debug) # Open the report by default if not args.no_browser: printInfo('Opening the HTML report...') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2)
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Get profile name profile_name = args.profile[0] # Search for AWS credentials credentials = read_creds(profile_name) if not credentials['AccessKeyId']: return 42 # Connect to IAM iam_client = connect_service('iam', credentials) if not iam_client: return 42 # Fetch the long-lived key ID if STS credentials are used if credentials['SessionToken']: akia_creds = read_creds(profile_name + '-nomfa') else: akia_creds = credentials aws_key_id = akia_creds['AccessKeyId'] aws_secret = akia_creds['SecretAccessKey'] # Fetch username printInfo('Fetching username...') user_name = get_username(credentials) # Create the new key try: # Create a new IAM key printInfo('Creating a new access key for \'%s\'...' % user_name) new_credentials = iam_client.create_access_key( UserName=user_name)['AccessKey'] show_access_keys(iam_client, user_name) except Exception as e: printException(e) return 42 # Save the new key if credentials['SessionToken']: write_creds_to_aws_credentials_file(profile_name + '-nomfa', new_credentials) else: write_creds_to_aws_credentials_file(profile_name, new_credentials) printInfo('Credentials file updated with new access key.') printInfo('Verifying access with the new key...') # Sleep because the access key may not be active server-side... time.sleep(5) if credentials['SessionToken']: new_credentials = read_creds(profile_name + '-nomfa') new_credentials = init_sts_session(profile_name, new_credentials) else: new_credentials = read_creds(profile_name) # Confirm that it works... try: new_iam_client = connect_service('iam', new_credentials) printInfo('Deleting the old access key...') new_iam_client.delete_access_key(AccessKeyId=aws_key_id, UserName=user_name) except Exception as e: printException(e) printInfo('Restoring your old credentials...') # Restore the old key here if credentials['SessionToken']: write_creds_to_aws_credentials_file(profile_name + '-nomfa', akia_creds) else: write_creds_to_aws_credentials_file(profile_name, akia_creds) return 42 # Summary of existing access keys try: show_access_keys(new_iam_client, user_name) printInfo('Success !') except Exception as e: printException(e) return 42
def main(): # Parse arguments parser = ListallArgumentParser() args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Support multiple environments for profile_name in args.profile: # Load the config try: report = Scout2Report(args.provider, profile_name, args.report_dir, args.timestamp) aws_config = report.jsrw.load_from_file(AWSCONFIG) services = aws_config['service_list'] except Exception as e: printException(e) printError( 'Error, failed to load the configuration for profile %s' % profile_name) continue # Create a ruleset with only whatever rules were specified... if args.config: rule_filename = args.config ruleset = TmpRuleset(rule_dirs=[os.getcwd()], rule_filename=args.config, rule_args=args.config_args) elif len(args.path) > 0: # Create a local tmp rule rule_dict = {'description': 'artifact'} rule_dict['path'] = args.path[0] rule_dict['conditions'] = [] rule_filename = 'listall-artifact.json' with open(os.path.join(os.getcwd(), rule_filename), 'wt') as f: f.write(json.dumps(rule_dict)) ruleset = TmpRuleset(rule_dirs=[os.getcwd()], rule_filename=rule_filename, rule_args=[]) else: printError( 'Error, you must provide either a rule configuration file or the path to the resources targeted.' ) continue # Process the rule pe = ProcessingEngine(ruleset) pe.run(aws_config, skip_dashboard=True) # Retrieve items rule = ruleset.rules[rule_filename][0] rule_service = rule.service.lower() rule_key = rule.key rule_type = rule.rule_type resources = aws_config['services'][rule_service][rule_type][rule_key][ 'items'] # Set the keys to output if len(args.keys): # 1. Explicitly provided on the CLI rule.keys = args.keys elif len(args.keys_file): # 2. Explicitly provided files that contain the list of keys rule.keys = [] for filename in args.keys_file: with open(filename, 'rt') as f: rule.keys += json.load(f)['keys'] else: try: # 3. Load default set of keys based on path target_path = rule.display_path if hasattr( rule, 'display_path') else rule.path listall_configs_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'output/data/listall-configs') target_file = os.path.join(listall_configs_dir, '%s.json' % target_path) if os.path.isfile(target_file): with open(target_file, 'rt') as f: rule.keys = json.load(f)['keys'] except: # 4. Print the object name rule.keys = ['name'] # Prepare the output format (lines, template) = format_listall_output(args.format_file[0], None, args.format, rule) # Print the output printInfo( generate_listall_output(lines, resources, aws_config, template, []))
def setUpClass(cls): configPrintException(True) creds = read_creds_from_environment_variables() cls.profile_name = 'travislike' if creds[ 'AccessKeyId'] == None else None cls.has_run_scout_suite = False
def setUpClass(cls): configPrintException(True) creds = read_creds_from_environment_variables() cls.profile_name = 'travislike' if creds['AccessKeyId'] == None else None cls.has_run_scout2 = False
def main(passed_args=None): """ Main method that runs a scan :return: """ # FIXME check that all requirements are installed # # Check version of opinel # requirements_file_path = '%s/requirements.txt' % os.path.dirname(sys.modules['__main__'].__file__) # if not check_requirements(requirements_file_path): # return 42 # Parse arguments parser = ScoutSuiteArgumentParser() if passed_args: args = parser.parse_args(passed_args) else: args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Create a cloud provider object cloud_provider = get_provider(provider=args.provider, profile=args.profile[0], project_id=args.project_id, folder_id=args.folder_id, organization_id=args.organization_id, report_dir=args.report_dir, timestamp=args.timestamp, services=args.services, skipped_services=args.skipped_services, thread_config=args.thread_config) if cloud_provider.provider_code == 'aws': if args.profile: report_file_name = 'aws-%s' % args.profile[0] else: report_file_name = 'aws' if cloud_provider.provider_code == 'gcp': if args.project_id: report_file_name = 'gcp-%s' % args.project_id elif args.organization_id: report_file_name = 'gcp-%s' % args.organization_id elif args.folder_id: report_file_name = 'gcp-%s' % args.folder_id else: report_file_name = 'gcp' if cloud_provider.provider_code == 'azure': report_file_name = 'azure' # Create a new report report = Scout2Report(args.provider, report_file_name, args.report_dir, args.timestamp) # Complete run, including pulling data from provider if not args.fetch_local: # Authenticate to the cloud provider authenticated = cloud_provider.authenticate(profile=args.profile[0], csv_credentials=args.csv_credentials, mfa_serial=args.mfa_serial, mfa_code=args.mfa_code, key_file=args.key_file, user_account=args.user_account, service_account=args.service_account, azure_cli=args.azure_cli, azure_msi=args.azure_msi, azure_service_principal=args.azure_service_principal, azure_file_auth=args.azure_file_auth, azure_user_credentials=args.azure_user_credentials) if not authenticated: return 42 # Fetch data from provider APIs try: cloud_provider.fetch(regions=args.regions) except KeyboardInterrupt: printInfo('\nCancelled by user') return 130 # Update means we reload the whole config and overwrite part of it if args.update: current_run_services = copy.deepcopy(cloud_provider.services) last_run_dict = report.jsrw.load_from_file(AWSCONFIG) cloud_provider.services = last_run_dict['services'] for service in cloud_provider.service_list: cloud_provider.services[service] = current_run_services[service] # Partial run, using pre-pulled data else: # Reload to flatten everything into a python dictionary last_run_dict = report.jsrw.load_from_file(AWSCONFIG) for key in last_run_dict: setattr(cloud_provider, key, last_run_dict[key]) # Pre processing cloud_provider.preprocessing(args.ip_ranges, args.ip_ranges_name_key) # Analyze config finding_rules = Ruleset(environment_name=args.profile[0], cloud_provider=args.provider, filename=args.ruleset, ip_ranges=args.ip_ranges, aws_account_id=cloud_provider.aws_account_id) processing_engine = ProcessingEngine(finding_rules) processing_engine.run(cloud_provider) # Create display filters filter_rules = Ruleset(cloud_provider=args.provider, filename='filters.json', rule_type='filters', aws_account_id=cloud_provider.aws_account_id) processing_engine = ProcessingEngine(filter_rules) processing_engine.run(cloud_provider) # Handle exceptions try: exceptions = RuleExceptions(args.profile[0], args.exceptions[0]) exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: printDebug('Warning, failed to load exceptions. The file may not exist or may have an invalid format.') exceptions = {} # Finalize cloud_provider.postprocessing(report.current_time, finding_rules) # TODO this is AWS-specific - move to postprocessing? # Get organization data if it exists try: profile = AWSProfiles.get(args.profile[0])[0] if 'source_profile' in profile.attributes: organization_info_file = os.path.join(os.path.expanduser('~/.aws/recipes/%s/organization.json' % profile.attributes['source_profile'])) if os.path.isfile(organization_info_file): with open(organization_info_file, 'rt') as f: org = {} accounts = json.load(f) for account in accounts: account_id = account.pop('Id') org[account_id] = account setattr(cloud_provider, 'organization', org) except Exception as e: pass # Save config and create HTML report html_report_path = report.save(cloud_provider, exceptions, args.force_write, args.debug) # Open the report by default if not args.no_browser: printInfo('Opening the HTML report...') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2) return 0
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('regions') parser.add_argument('partition-name') parser.add_argument('bucket-name') parser.parser.add_argument('--aws-account-id', dest='aws_account_id', default=[ None ], nargs='+', help='Bleh.') parser.parser.add_argument('--from', dest='from_date', default=[ None ], nargs='+', help='Bleh.') parser.parser.add_argument('--to', dest='to_date', default=[ None ], nargs='+', help='Bleh.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Arguments profile_name = args.profile[0] try: from_date = datetime.datetime.strptime(args.from_date[0], "%Y/%m/%d").date() to_date = datetime.datetime.strptime(args.to_date[0], "%Y/%m/%d").date() delta = to_date - from_date except Exception as e: printException(e) printError('Error: dates must be formatted of the following format YYYY/MM/DD') return 42 if delta.days < 0: printError('Error: your \'to\' date is earlier than your \'from\' date') return 42 # Search for AWS credentials credentials = read_creds(profile_name) if not credentials['AccessKeyId']: return 42 # Fetch AWS account ID if not args.aws_account_id[0]: printInfo('Fetching the AWS account ID...') aws_account_id = get_aws_account_id(credentials) else: aws_account_id = args.aws_account_id[0] global cloudtrail_log_path cloudtrail_log_path = cloudtrail_log_path.replace('AWS_ACCOUNT_ID', aws_account_id) # Create download dir if not os.path.exists(download_folder): os.makedirs(download_folder) # Iterate through regions s3_clients = {} for region in build_region_list('cloudtrail', args.regions, args.partition_name): # Connect to CloudTrail cloudtrail_client = connect_service('cloudtrail', credentials, region) if not cloudtrail_client: continue # Get information about the S3 bucket that receives CloudTrail logs trails = cloudtrail_client.describe_trails() for trail in trails['trailList']: bucket_name = trail['S3BucketName'] prefix = trail['S3KeyPrefix'] if 'S3KeyPrefix' in trail else '' # Connect to S3 manage_dictionary(s3_clients, region, connect_service('s3', credentials, region)) target_bucket_region = get_s3_bucket_location(s3_clients[region], bucket_name) manage_dictionary(s3_clients, target_bucket_region, connect_service('s3', credentials, target_bucket_region)) s3_client = s3_clients[target_bucket_region] # Generate base path for files log_path = os.path.join(prefix, cloudtrail_log_path.replace('REGION', region)) # Download files printInfo('Downloading log files in %s... ' % region, False) keys = [] for i in range(delta.days + 1): day = from_date + timedelta(days=i) folder_path = os.path.join(log_path, day.strftime("%Y/%m/%d")) try: objects = handle_truncated_response(s3_client.list_objects, {'Bucket': bucket_name, 'Prefix': folder_path}, ['Contents']) for o in objects['Contents']: keys.append([o['Key'], 0]) except Exception as e: printException(e) pass thread_work(keys, download_object, params = {'Bucket': bucket_name, 'S3Client': s3_client}, num_threads = 100) printInfo('Done') # Iterate through files and gunzip 'em printInfo('Decompressing files...') gzlogs = [] for root, dirnames, filenames in os.walk(download_folder): for filename in filenames: gzlogs.append(filename) thread_work(gzlogs, gunzip_file, num_threads = 30)
def main(args): setup_logging(args.debug) if args.version: print aws_inventory.__version__ return api_model = build_api_model() if args.list_svcs: print '\n'.join(sorted(filter_services(api_model))) return # configure the debug level for opinel configPrintException(args.debug) # validate services against API mode # available_services = api_model.keys() if args.services: invalid_included_services = [ svc for svc in args.services if svc not in available_services ] if invalid_included_services: raise EnvironmentError('Invalid service(s) specified: {}'.format( ', '.join(invalid_included_services))) if args.excluded_services: invalid_excluded_services = [ svc for svc in args.excluded_services if svc not in available_services ] if invalid_excluded_services: raise EnvironmentError('Invalid service(s) to exclude: {}'.format( ', '.join(invalid_excluded_services))) # validate regions against API model # if args.regions: available_regions = set() for svc in available_services: available_regions.update(api_model[svc]['regions']) invalid_regions = [ region for region in args.regions if region not in available_regions ] if invalid_regions: raise EnvironmentError('Invalid region(s) specified: {}'.format( ', '.join(invalid_regions))) # create the list of services to analyze services = filter_services(api_model, frozenset(args.services), frozenset(args.excluded_services)) if not services: raise EnvironmentError('List of AWS services to be analyzed is empty.') LOGGER.debug('%d AWS service(s) to inspect: %s.', len(services), ', '.join(services)) op_blacklist_parser = aws_inventory.blacklist.OpBlacklistParser( args.op_blacklist, api_model) service_descriptors = filter_operations(api_model, op_blacklist_parser, args.regions, services) if not service_descriptors: raise EnvironmentError( 'No operations to invoke for specifed AWS services and regions.') ops_count = 0 for svc_name in service_descriptors: ops_count += (len(service_descriptors[svc_name]['ops']) * len(service_descriptors[svc_name]['regions'])) if args.list_operations: print '[{}]\n{}\n'.format( svc_name, '\n'.join(service_descriptors[svc_name]['ops']) or '# NONE') if args.list_operations: print 'Total operations to invoke: {}'.format(ops_count) else: LOGGER.debug('Total operations to invoke: %d.', ops_count) aws_inventory.invoker.ApiInvoker(args, service_descriptors, ops_count).start()
def main(): # Parse arguments parser = Scout2ArgumentParser() args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Set the profile name profile_name = args.profile[0] # Search for AWS credentials if not args.fetch_local: credentials = read_creds(args.profile[0], args.csv_credentials, args.mfa_serial, args.mfa_code) if credentials['AccessKeyId'] is None: return 42 # Create a new Scout2 config report = Scout2Report(profile_name, args.report_dir, args.timestamp) aws_config = Scout2Config(profile_name, args.report_dir, args.timestamp, args.services, args.skipped_services, args.thread_config) if not args.fetch_local: # Fetch data from AWS APIs if not running a local analysis try: aws_config.fetch(credentials, regions=args.regions, partition_name = get_partition_name(credentials)) except KeyboardInterrupt: printInfo('\nCancelled by user') return 130 aws_config = report.jsrw.to_dict(aws_config) # Set the account ID aws_config['aws_account_id'] = get_aws_account_id(credentials) # Update means we reload the whole config and overwrite part of it if args.update == True: new_aws_config = copy.deepcopy(aws_config) aws_config = report.jsrw.load_from_file(AWSCONFIG) for service in new_aws_config['service_list']: # Per service only for now, may add per region & per VPC later... aws_config['services'][service] = new_aws_config['services'][service] # Update the metadata too aws_config['metadata'] = Scout2Config('default', None, None, [], []).metadata else: # Reload to flatten everything into a python dictionary aws_config = report.jsrw.load_from_file(AWSCONFIG) # Pre processing preprocessing(aws_config, args.ip_ranges, args.ip_ranges_name_key) # Analyze config finding_rules = Ruleset(profile_name, filename = args.ruleset, ip_ranges = args.ip_ranges, aws_account_id = aws_config['aws_account_id']) pe = ProcessingEngine(finding_rules) pe.run(aws_config) # Create display filters filter_rules = Ruleset(filename = 'filters.json', rule_type = 'filters', aws_account_id = aws_config['aws_account_id']) pe = ProcessingEngine(filter_rules) pe.run(aws_config) # Handle exceptions try: exceptions = RuleExceptions(profile_name, args.exceptions[0]) exceptions.process(aws_config) exceptions = exceptions.exceptions except Exception as e: printDebug('Warning, failed to load exceptions. The file may not exist or may have an invalid format.') exceptions = {} # Finalize postprocessing(aws_config, report.current_time, finding_rules) # Get organization data if it exists try: profile = AWSProfiles.get(profile_name)[0] if 'source_profile' in profile.attributes: organization_info_file = os.path.join(os.path.expanduser('~/.aws/recipes/%s/organization.json' % profile.attributes['source_profile'])) if os.path.isfile(organization_info_file): with open(organization_info_file, 'rt') as f: org = {} accounts = json.load(f) for account in accounts: account_id = account.pop('Id') org[account_id] = account aws_config['organization'] = org except: pass # Save config and create HTML report html_report_path = report.save(aws_config, exceptions, args.force_write, args.debug) # Open the report by default if not args.no_browser: printInfo('Opening the HTML report...') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2) return 0