def createAWSClient(service, configDict=None): """Creates a boto3 client object for communicating with a specific AWS service. This is always invoked by the lambda run/deployment scripts to create a client to the Lambda service, but can also be invoked within the Lambda function to connect to other AWS services. Arguments: service {string} -- The AWS service to create a connection to. Keyword Arguments: configDict {string} -- AWS Configuration details. If None/not provided details will be loaded from default config.yaml file (default: {None}) Returns: [boto3.client] -- A client object that can be used to invoke various services from the associated AWS service. """ if configDict is None: configDict = loadEnvFile(None, None) clientKwargs = {'region_name': configDict['region']} if ('aws_access_key_id' in configDict and configDict['aws_access_key_id'] is not None): clientKwargs['aws_access_key_id'] = configDict['aws_access_key_id'] clientKwargs['aws_secret_access_key'] = configDict[ 'aws_secret_access_key'] # noqa: E501 lambdaClient = boto3.client(service, **clientKwargs) return lambdaClient
def setEnvVars(runType): # Load env variables from relevant .yaml file envDict, envLines = loadEnvFile(runType, 'config/{}.yaml') # If no environemnt variables are set, do nothing to config if 'environment_variables' not in envDict: shutil.copyfile('config.yaml', 'run_config.yaml') return # Overwrite/add any vars in the core config.yaml file configDict, configLines = loadEnvFile(runType, None) envVars = configDict['environment_variables'] for key, value in envDict['environment_variables'].items(): envVars[key] = value newEnvVars = yaml.dump({'environment_variables': envVars}, default_flow_style=False) try: with open('run_config.yaml', 'w') as newConfig: write = True written = False for line in configLines: if line.strip() == '# === END_ENV_VARIABLES ===': write = True if write is False and written is False: newConfig.write(newEnvVars) written = True elif write is True: newConfig.write(line) if line.strip() == '# === START_ENV_VARIABLES ===': write = False except IOError as err: logger.error(('Script lacks necessary permissions, ' 'ensure user has permission to write to directory')) raise err
def createAWSClient(service, configDict=None): if configDict is None: configDict, configLines = loadEnvFile(None, None) clientKwargs = {'region_name': configDict['region']} if ('aws_access_key_id' in configDict and configDict['aws_access_key_id'] is not None): clientKwargs['aws_access_key_id'] = configDict['aws_access_key_id'] clientKwargs['aws_secret_access_key'] = configDict[ 'aws_secret_access_key'] # noqa: E501 lambdaClient = boto3.client(service, **clientKwargs) return lambdaClient
def createEventMapping(runType): logger.info('Creating event Source mappings for Lambda') try: with open('config/event_sources_{}.json'.format(runType)) as sources: try: eventMappings = json.load(sources) except json.decoder.JSONDecodeError as err: logger.error('Unable to parse JSON file') raise err except FileNotFoundError: logger.info('No Event Source mapping provided') return except IOError as err: logger.error('Unable to open JSON file') raise err if len(eventMappings['EventSourceMappings']) < 1: logger.info('No event sources defined') return configDict, configLines = loadEnvFile(runType, None) lambdaClient = createAWSClient('lambda', configDict) for mapping in eventMappings['EventSourceMappings']: logger.debug('Adding event source mapping for function') createKwargs = { 'EventSourceArn': mapping['EventSourceArn'], 'FunctionName': configDict['function_name'], 'Enabled': mapping['Enabled'], 'BatchSize': mapping['BatchSize'], } if 'StartingPosition' in mapping: createKwargs['StartingPosition'] = mapping['StartingPosition'] if mapping['StartingPosition'] == 'AT_TIMESTAMP': createKwargs['StartingPositionTimestamp'] = mapping[ 'StartingPositionTimestamp'] # noqa: E50 try: lambdaClient.create_event_source_mapping(**createKwargs) except lambdaClient.exceptions.ResourceConflictException as err: logger.info('Event Mapping already exists, update') logger.debug(err) updateEventMapping(lambdaClient, mapping, configDict)
def test_read_env_failure(self, mock_yaml): try: loadEnvFile('development', 'config/{}.yaml') except YAMLError: pass self.assertRaises(YAMLError)
def test_load_empty_env(self, mock_yaml): resDict = loadEnvFile('development', None) self.assertEqual(resDict, {})
def test_load_env_failure(self, mock_yaml): try: loadEnvFile('development', 'missing/{}.yaml') except FileNotFoundError: pass self.assertRaises(FileNotFoundError)
def test_load_env_config_success(self, mock_yaml): resDict = loadEnvFile('development', 'config/{}.yaml') self.assertTrue(resDict['testing'])
def test_load_env_success(self, mock_yaml): resDict = loadEnvFile('development', None) self.assertTrue(resDict['testing'])
def test_load_env_config_success(self, mock_yaml): m = mock_open() with patch('builtins.open', m, create=True): resDict, resLines = loadEnvFile('development', 'config/{}.yaml') self.assertTrue(resDict['testing'])
def test_load_env_config_success(self, mock_yaml): with patch('builtins.open'): resDict = loadEnvFile('testing', 'config/{}.yaml') self.assertTrue(resDict['testing'])
def test_read_env_failure(self, mock_yaml): with self.assertRaises(YAMLError): loadEnvFile('development', 'config/{}.yaml')
def test_load_env_failure(self, mock_yaml): resDict = loadEnvFile('development', 'missing/{}.yaml') self.assertEqual(resDict, {})