def create_reduce_function(function_id, registry, run_registry=None):
function_name = "red-{}".format(function_id)
deploy_info = create_deploy_info(
settings.REDUCE_PATH,
settings.REDUCE_CONFIG_TEMPLATE_PATH,
registry,
run_registry
)
# create function
function = Function(function_name, deploy_info, debug=settings.DEBUG)
# load config data
config_data = function.get_config_data()
spec = config_data['spec']
if spec['env'] == None:
spec['env'] = []
# update env vars
update_env_var(spec['env'], name="ID", value=str(function_id))
update_env_var(spec['env'], name=settings.HDFS_HOST_KEY, value=settings.HDFS_HOST_VALUE)
output_dir = "" if settings.HDFS_OUTPUT_DIR_VALUE == "/" else settings.HDFS_OUTPUT_DIR_VALUE
update_env_var(spec['env'], name='REDUCER_OUTPUT_FILENAME', value="{}/out_{}.txt".format(output_dir, function_id))
update_env_var(spec['env'], name=settings.HDFS_CHUNK_COUNT_KEY, value=settings.HDFS_CHUNK_COUNT_VALUE)
update_env_var(spec['env'], name=settings.NUM_MAPPERS_KEY, value=settings.NUM_MAPPERS_VALUE)
update_env_var(spec['env'], name=settings.HDFS_USER_KEY, value=settings.HDFS_USER_VALUE)
update_env_var(spec['env'], name=settings.DONE_TOPIC_KEY, value=settings.DONE_TOPIC_VALUE)
update_env_var(spec['env'], name=settings.RMQ_HOST_KEY, value=settings.RMQ_HOST_VALUE)
update_env_var(spec['env'], name=settings.RMQ_PORT_KEY, value=settings.RMQ_PORT_VALUE)
update_env_var(spec['env'], name=settings.RMQ_USER_KEY, value=settings.RMQ_USER_VALUE)
update_env_var(spec['env'], name=settings.RMQ_PASS_KEY, value=settings.RMQ_PASS_VALUE)
update_env_var(spec['env'], name=settings.EXCHANGE_NAME_KEY, value=settings.EXCHANGE_NAME_VALUE)
# update rmq trigger info
update_rmq_trigger(
spec['triggers'],
settings.RMQ_URL_VALUE,
settings.EXCHANGE_NAME_VALUE,
["{}{}".format(settings.REDUCE_TOPIC_PREFIX_VALUE, function_id)],
max_workers=3,
)
# update http trigger
update_http_trigger(
spec['triggers'],
settings.REDUCE_PORT_START + function_id,
max_workers=3,
)
# apply config changes
function.apply_config(config_data)
return function
def create_map_function(function_id, registry, run_registry=None):
function_name = "map-{}".format(function_id)
deploy_info = create_deploy_info(
settings.MAP_PATH,
settings.MAP_CONFIG_TEMPLATE_PATH,
registry,
run_registry
)
# create function
function = Function(function_name, deploy_info, debug=settings.DEBUG)
# load config data
config_data = function.get_config_data()
spec = config_data['spec']
if spec['env'] == None:
spec['env'] = []
# update env vars
update_env_var(spec['env'], name="ID", value=str(function_id))
update_env_var(spec['env'], name=settings.RMQ_HOST_KEY, value=settings.RMQ_HOST_VALUE)
update_env_var(spec['env'], name=settings.RMQ_PORT_KEY, value=settings.RMQ_PORT_VALUE)
update_env_var(spec['env'], name=settings.RMQ_USER_KEY, value=settings.RMQ_USER_VALUE)
update_env_var(spec['env'], name=settings.RMQ_PASS_KEY, value=settings.RMQ_PASS_VALUE)
update_env_var(spec['env'], name=settings.NUM_REDUCERS_KEY, value=settings.NUM_REDUCERS_VALUE)
update_env_var(spec['env'], name=settings.REDUCE_TOPIC_PREFIX_KEY, value=settings.REDUCE_TOPIC_PREFIX_VALUE)
update_env_var(spec['env'], name=settings.EXCHANGE_NAME_KEY, value=settings.EXCHANGE_NAME_VALUE)
update_env_var(spec['env'], name=settings.HDFS_HOST_KEY, value=settings.HDFS_HOST_VALUE)
update_env_var(spec['env'], name=settings.DONE_TOPIC_KEY, value=settings.DONE_TOPIC_VALUE)
update_env_var(spec['env'], name=settings.HDFS_USER_KEY, value=settings.HDFS_USER_VALUE)
# update rmq trigger info
update_rmq_trigger(
spec['triggers'],
settings.RMQ_URL_VALUE,
settings.EXCHANGE_NAME_VALUE,
["{}{}".format(settings.MAP_TOPIC_PREFIX_VALUE, function_id)],
max_workers=3
)
# update http trigger
update_http_trigger(
spec['triggers'],
settings.MAP_PORT_START + function_id,
max_workers=3,
)
# apply config changes
function.apply_config(config_data)
return function
