def create_reduce_function(function_id, registry, run_registry=None): function_name = "red-{}".format(function_id) deploy_info = create_deploy_info( settings.REDUCE_PATH, settings.REDUCE_CONFIG_TEMPLATE_PATH, registry, run_registry ) # create function function = Function(function_name, deploy_info, debug=settings.DEBUG) # load config data config_data = function.get_config_data() spec = config_data['spec'] if spec['env'] == None: spec['env'] = [] # update env vars update_env_var(spec['env'], name="ID", value=str(function_id)) update_env_var(spec['env'], name=settings.HDFS_HOST_KEY, value=settings.HDFS_HOST_VALUE) output_dir = "" if settings.HDFS_OUTPUT_DIR_VALUE == "/" else settings.HDFS_OUTPUT_DIR_VALUE update_env_var(spec['env'], name='REDUCER_OUTPUT_FILENAME', value="{}/out_{}.txt".format(output_dir, function_id)) update_env_var(spec['env'], name=settings.HDFS_CHUNK_COUNT_KEY, value=settings.HDFS_CHUNK_COUNT_VALUE) update_env_var(spec['env'], name=settings.NUM_MAPPERS_KEY, value=settings.NUM_MAPPERS_VALUE) update_env_var(spec['env'], name=settings.HDFS_USER_KEY, value=settings.HDFS_USER_VALUE) update_env_var(spec['env'], name=settings.DONE_TOPIC_KEY, value=settings.DONE_TOPIC_VALUE) update_env_var(spec['env'], name=settings.RMQ_HOST_KEY, value=settings.RMQ_HOST_VALUE) update_env_var(spec['env'], name=settings.RMQ_PORT_KEY, value=settings.RMQ_PORT_VALUE) update_env_var(spec['env'], name=settings.RMQ_USER_KEY, value=settings.RMQ_USER_VALUE) update_env_var(spec['env'], name=settings.RMQ_PASS_KEY, value=settings.RMQ_PASS_VALUE) update_env_var(spec['env'], name=settings.EXCHANGE_NAME_KEY, value=settings.EXCHANGE_NAME_VALUE) # update rmq trigger info update_rmq_trigger( spec['triggers'], settings.RMQ_URL_VALUE, settings.EXCHANGE_NAME_VALUE, ["{}{}".format(settings.REDUCE_TOPIC_PREFIX_VALUE, function_id)], max_workers=3, ) # update http trigger update_http_trigger( spec['triggers'], settings.REDUCE_PORT_START + function_id, max_workers=3, ) # apply config changes function.apply_config(config_data) return function
def create_map_function(function_id, registry, run_registry=None): function_name = "map-{}".format(function_id) deploy_info = create_deploy_info( settings.MAP_PATH, settings.MAP_CONFIG_TEMPLATE_PATH, registry, run_registry ) # create function function = Function(function_name, deploy_info, debug=settings.DEBUG) # load config data config_data = function.get_config_data() spec = config_data['spec'] if spec['env'] == None: spec['env'] = [] # update env vars update_env_var(spec['env'], name="ID", value=str(function_id)) update_env_var(spec['env'], name=settings.RMQ_HOST_KEY, value=settings.RMQ_HOST_VALUE) update_env_var(spec['env'], name=settings.RMQ_PORT_KEY, value=settings.RMQ_PORT_VALUE) update_env_var(spec['env'], name=settings.RMQ_USER_KEY, value=settings.RMQ_USER_VALUE) update_env_var(spec['env'], name=settings.RMQ_PASS_KEY, value=settings.RMQ_PASS_VALUE) update_env_var(spec['env'], name=settings.NUM_REDUCERS_KEY, value=settings.NUM_REDUCERS_VALUE) update_env_var(spec['env'], name=settings.REDUCE_TOPIC_PREFIX_KEY, value=settings.REDUCE_TOPIC_PREFIX_VALUE) update_env_var(spec['env'], name=settings.EXCHANGE_NAME_KEY, value=settings.EXCHANGE_NAME_VALUE) update_env_var(spec['env'], name=settings.HDFS_HOST_KEY, value=settings.HDFS_HOST_VALUE) update_env_var(spec['env'], name=settings.DONE_TOPIC_KEY, value=settings.DONE_TOPIC_VALUE) update_env_var(spec['env'], name=settings.HDFS_USER_KEY, value=settings.HDFS_USER_VALUE) # update rmq trigger info update_rmq_trigger( spec['triggers'], settings.RMQ_URL_VALUE, settings.EXCHANGE_NAME_VALUE, ["{}{}".format(settings.MAP_TOPIC_PREFIX_VALUE, function_id)], max_workers=3 ) # update http trigger update_http_trigger( spec['triggers'], settings.MAP_PORT_START + function_id, max_workers=3, ) # apply config changes function.apply_config(config_data) return function