def spark_mapper(current_range):
"""
Gets the paths to the file(s) in the current executor, then
declares the headers found.
Args:
current_range (tuple): A pair that contains the starting and
ending values of the current range.
Returns:
function: The map function to be executed on each executor,
complete with all headers needed for the analysis.
"""
# Get and declare headers on each worker
headers_on_executor = [
SparkFiles.get(ntpath.basename(filepath))
for filepath in includes_headers
]
Utils.declare_headers(headers_on_executor)
# Get and declare shared libraries on each worker
shared_libs_on_ex = [
SparkFiles.get(ntpath.basename(filepath))
for filepath in includes_shared_libraries
]
Utils.declare_shared_libraries(shared_libs_on_ex)
return mapper(current_range)
def include_shared_libraries(shared_libraries_paths):
"""
Includes the C++ shared libraries to be declared before execution.
Each library is also declared on the current running session. If any pcm
file is present in the same folder as the shared libraries, the function
will try to retrieve them (and distribute them if working on a distributed
backend).
Args:
shared_libraries_paths (str, iter): A string or an iterable (such as a
list, set...) containing the paths to all necessary C++ shared
libraries as strings. This function accepts both paths to the
libraries themselves and paths to directories containing the
libraries.
"""
global current_backend, includes_shared_libraries
libraries_to_include = set()
pcm_to_include = set()
if isinstance(shared_libraries_paths, str):
pcm_to_include, libraries_to_include = _check_pcm_in_library_path(
shared_libraries_paths
)
else:
for path_string in shared_libraries_paths:
pcm, libraries = _check_pcm_in_library_path(
path_string
)
libraries_to_include.update(libraries)
pcm_to_include.update(pcm)
# If not on the local backend, distribute files to executors
if not isinstance(current_backend, Local):
current_backend.distribute_files(libraries_to_include)
current_backend.distribute_files(pcm_to_include)
# Declare the shared libraries in ROOT
Utils.declare_shared_libraries(libraries_to_include)
# Finally, add everything to the includes set
includes_shared_libraries.update(includes_shared_libraries)