def merge_conf(conf: Dict, yaml: Dict) -> Dict:
result = yaml.copy()
# We have to merge the default config with all the defined functions
for provider in FAAS_PROVIDERS:
for index, function in enumerate(result.get('functions', {}).get(provider, {})):
result['functions'][provider][index] = \
DataTypesUtils.merge_dicts_with_copy(conf.get(provider,{}), function)
result['scar'] = DataTypesUtils.merge_dicts_with_copy(result.get('scar', {}),
conf.get('scar', {}))
return result
def merge_cmd_yaml(cmd: Dict, yaml: Dict) -> Dict:
result = yaml.copy()
# We merge the cli commands with all the defined functions
# CLI only allows define AWS parameters
for cli_cmd in cmd.get('functions', {}).get("aws", {}):
for index, function in enumerate(result.get('functions', {}).get("aws", {})):
result['functions']['aws'][index] = \
DataTypesUtils.merge_dicts_with_copy(function, cli_cmd)
result['scar'] = DataTypesUtils.merge_dicts_with_copy(result.get('scar', {}),
cmd.get('scar', {}))
result['storage_providers'] = DataTypesUtils.merge_dicts_with_copy(result.get('storage_providers', {}),
cmd.get('storage_providers', {}))
return result
def parse_aws_function(self, function_name, function_data):
aws_args = {}
scar_args = {}
# Get function name
aws_args['lambda'] = self.parse_lambda_args(function_data)
aws_args['lambda']['name'] = function_name
aws_services = ['iam', 'cloudwatch', 's3', 'api_gateway', 'batch']
aws_args.update(DataTypesUtils.parse_arg_list(aws_services, function_data))
other_args = [('profile', 'boto_profile'), 'region', 'execution_mode']
aws_args.update(DataTypesUtils.parse_arg_list(other_args, function_data))
scar_args.update(DataTypesUtils.parse_arg_list(['supervisor_version'], function_data))
scar = {'scar': scar_args if scar_args else {}}
aws = {'aws': aws_args if aws_args else {}}
return DataTypesUtils.merge_dicts(scar, aws)
def _parse_s3_args(aws_args: Dict, cmd_args: Dict) -> Dict:
s3_arg_list = [
'deployment_bucket', 'input_bucket', 'output_bucket',
('bucket', 'input_bucket')
]
s3_args = DataTypesUtils.parse_arg_list(s3_arg_list, cmd_args)
storage = {}
if s3_args:
if 'deployment_bucket' in s3_args:
aws_args['lambda']['deployment'] = {
'bucket': s3_args['deployment_bucket']
}
if 'input_bucket' in s3_args:
aws_args['lambda']['input'] = [{
'storage_provider': 's3',
'path': s3_args['input_bucket']
}]
if 'output_bucket' in s3_args:
aws_args['lambda']['output'] = [{
'storage_provider': 's3',
'path': s3_args['output_bucket']
}]
storage['storage_providers'] = {'s3': {}}
return storage
def process_asynchronous_lambda_invocations(self, s3_event_list):
if (len(s3_event_list) > MAX_CONCURRENT_INVOCATIONS):
s3_file_chunk_list = DataTypesUtils.divide_list_in_chunks(s3_event_list, MAX_CONCURRENT_INVOCATIONS)
for s3_file_chunk in s3_file_chunk_list:
self._launch_concurrent_lambda_invocations(s3_file_chunk)
else:
self._launch_concurrent_lambda_invocations(s3_event_list)
def parse_arguments(self):
"""
Merge the scar.conf parameters, the cmd parameters and the yaml
file parameters in a single dictionary.
The precedence of parameters is CMD >> YAML >> SCAR.CONF
That is, the CMD parameter will override any other configuration,
and the YAML parameters will override the SCAR.CONF settings
"""
merged_args = ConfigFileParser().get_properties()
cmd_args = CommandParser(self).parse_arguments()
if 'conf_file' in cmd_args['scar'] and cmd_args['scar']['conf_file']:
yaml_args = YamlParser(cmd_args['scar']).parse_arguments()
merged_args = DataTypesUtils.merge_dicts(yaml_args, merged_args)
merged_args = DataTypesUtils.merge_dicts(cmd_args, merged_args)
self.cloud_provider.parse_arguments(**merged_args)
merged_args['scar']['func']()
def _parse_aws_args(cmd_args: Dict) -> Dict:
aws_args = {}
other_args = [('profile', 'boto_profile'), 'region', 'execution_mode']
_set_args(aws_args, 'iam', _parse_iam_args(cmd_args))
_set_args(aws_args, 'lambda', _parse_lambda_args(cmd_args))
_set_args(aws_args, 'batch', _parse_batch_args(cmd_args))
_set_args(aws_args, 'cloudwatch', _parse_cloudwatchlogs_args(cmd_args))
_set_args(aws_args, 'api_gateway', _parse_api_gateway_args(cmd_args))
aws_args.update(DataTypesUtils.parse_arg_list(other_args, cmd_args))
storage = _parse_s3_args(aws_args, cmd_args)
result = {'functions': {'aws': [aws_args]}}
if storage:
result.update(storage)
return result
def _parse_lambda_args(cmd_args: Dict) -> Dict:
lambda_arg_list = [
'name', 'asynchronous', 'init_script', 'run_script', 'c_args',
'memory', 'timeout', 'timeout_threshold', 'image', 'image_file',
'description', 'lambda_role', 'extra_payload',
('environment', 'environment_variables'), 'layers',
'lambda_environment', 'list_layers', 'log_level', 'preheat'
]
lambda_args = DataTypesUtils.parse_arg_list(lambda_arg_list, cmd_args)
# Standardize log level if defined
if "log_level" in lambda_args:
lambda_args['log_level'] = lambda_args['log_level'].upper()
# Parse environment variables
lambda_args.update(_get_lambda_environment_variables(lambda_args))
return lambda_args
def parse_arguments(self) -> Dict:
"""Command parsing and selection"""
try:
cmd_args = self.parser.parse_args()
if cmd_args.version:
print(f"SCAR {version.__version__}")
sys.exit(0)
cmd_args = vars(cmd_args)
if 'func' not in cmd_args:
raise excp.MissingCommandError()
scar_args = _parse_scar_args(cmd_args)
aws_args = _parse_aws_args(cmd_args)
return cmd_args['func'], DataTypesUtils.merge_dicts_with_copy(
scar_args, aws_args)
except AttributeError as aerr:
logger.error(
"Incorrect arguments: use scar -h to see the options available",
f"Error parsing arguments: {aerr}")
except:
print("Unexpected error:", sys.exc_info()[0])
raise
def _parse_iam_args(cmd_args: Dict) -> Dict:
iam_args = [('iam_role', 'role')]
return DataTypesUtils.parse_arg_list(iam_args, cmd_args)
def _parse_scar_args(cmd_args: Dict) -> Dict:
scar_args = [
'conf_file', 'json', 'verbose', 'path', 'execution_mode',
'output_file', 'supervisor_version', 'all'
]
return {'scar': DataTypesUtils.parse_arg_list(scar_args, cmd_args)}
def _parse_api_gateway_args(cmd_args: Dict) -> Dict:
api_gtw_args = [('api_gateway_name', 'name'), 'parameters', 'data_binary',
'json_data']
return DataTypesUtils.parse_arg_list(api_gtw_args, cmd_args)
def _parse_cloudwatchlogs_args(cmd_args: Dict) -> Dict:
cw_log_args = ['log_stream_name', 'request_id']
return DataTypesUtils.parse_arg_list(cw_log_args, cmd_args)
def _parse_batch_args(cmd_args: Dict) -> Dict:
batch_args = [('batch_vcpus', 'vcpus'), ('batch_memory', 'memory'),
'enable_gpu']
return DataTypesUtils.parse_arg_list(batch_args, cmd_args)
def parse_lambda_args(self, cmd_args):
lambda_args = ['asynchronous', 'init_script', 'run_script', 'c_args', 'memory', 'time',
'timeout_threshold', 'log_level', 'image', 'image_file', 'description',
'lambda_role', 'extra_payload', ('environment', 'environment_variables'),
'layers', 'lambda_environment']
return DataTypesUtils.parse_arg_list(lambda_args, cmd_args)