def host_dump(client, payload, target_filename, host_view=None, pre_warm=True):
"""Send payload to each host and dump it on the filesystem
Nothing is done in case the file already exists.
The payload is shipped only once per node in the cluster.
"""
if host_view is None:
host_view = get_host_view(client)
client = host_view.client
@interactive
def dump_payload(payload, filename):
from sklearn.externals import joblib
import os
folder = os.path.dirname(filename)
if not os.path.exists(folder):
os.makedirs(folder)
return joblib.dump(payload, filename)
missing_ids = _missing_file_engine_ids(host_view, target_filename)
if missing_ids:
first_id = missing_ids[0]
# Do a first dispatch to the first node to avoid concurrent write in
# case of shared filesystem
client[first_id].apply_sync(dump_payload, payload, target_filename)
# Refetch the list of engine ids where the file is missing
missing_ids = _missing_file_engine_ids(host_view, target_filename)
# Restrict the view to hosts where the target data file is still
# missing for the final dispatch
client[missing_ids].apply_sync(dump_payload, payload, target_filename)
if pre_warm:
warm_mmap(client, [target_filename], host_view=host_view)
def host_dump(client, payload, target_filename, host_view=None, pre_warm=True):
"""Send payload to each host and dump it on the filesystem
Nothing is done in case the file already exists.
The payload is shipped only once per node in the cluster.
"""
if host_view is None:
host_view = get_host_view(client)
client = host_view.client
@interactive
def dump_payload(payload, filename):
from sklearn.externals import joblib
import os
folder = os.path.dirname(filename)
if not os.path.exists(folder):
os.makedirs(folder)
return joblib.dump(payload, filename)
missing_ids = _missing_file_engine_ids(host_view, target_filename)
if missing_ids:
first_id = missing_ids[0]
# Do a first dispatch to the first node to avoid concurrent write in
# case of shared filesystem
client[first_id].apply_sync(dump_payload, payload, target_filename)
# Refetch the list of engine ids where the file is missing
missing_ids = _missing_file_engine_ids(host_view, target_filename)
# Restrict the view to hosts where the target data file is still
# missing for the final dispatch
client[missing_ids].apply_sync(dump_payload, payload, target_filename)
if pre_warm:
warm_mmap(client, [target_filename], host_view=host_view)
def warm_mmap(client, data_filenames, host_view=None):
"""Trigger a disk load on all the arrays data_filenames.
Assume the files are shared on all the hosts using NFS or
have been previously been dumped there with the host_dump function.
"""
if host_view is None:
host_view = get_host_view(client)
# Second step: for each data file and host, mmap the arrays of the file
# and trigger a sequential read of all the arrays' data
@interactive
def load_in_memory(filenames):
from sklearn.externals import joblib
for filename in filenames:
arrays = joblib.load(filename, mmap_mode='r')
for array in arrays:
if hasattr(array, 'max'):
array.max() # trigger the disk read
data_filenames = [os.path.abspath(f) for f in data_filenames]
host_view.apply_sync(load_in_memory, data_filenames)
def warm_mmap(client, data_filenames, host_view=None):
"""Trigger a disk load on all the arrays data_filenames.
Assume the files are shared on all the hosts using NFS or
have been previously been dumped there with the host_dump function.
"""
if host_view is None:
host_view = get_host_view(client)
# Second step: for each data file and host, mmap the arrays of the file
# and trigger a sequential read of all the arrays' data
@interactive
def load_in_memory(filenames):
from sklearn.externals import joblib
for filename in filenames:
arrays = joblib.load(filename, mmap_mode='r')
for array in arrays:
if hasattr(array, 'max'):
array.max() # trigger the disk read
data_filenames = [os.path.abspath(f) for f in data_filenames]
host_view.apply_sync(load_in_memory, data_filenames)