def parse_module_dependencies(self,
func,
hint,
from_shared_storage=False,
sync_to_shared_storage=False):
"""
:param func: the function to prase
:param hint: user defined key
:param from_shared_storage: whether fetch this module dependencies data from shared storage
:param sync_to_shared_storage: whether sync this module dependencies data from shared storage
:rtype: a module_dependencies_key that can be use in map function
Usage
>>> pwex.parse_module_dependencies(foo, hint)
"""
assert hint is not None
assert not (from_shared_storage and sync_to_shared_storage)
user_key = hint
if from_shared_storage:
return
start = time.time()
func_str, mod_paths = self.serializer([func])
func_str = func_str[0]
module_data = create_mod_data(mod_paths)
func_module_str = pickle.dumps(
{
'func': func_str,
'module_data': module_data
}, -1)
end = time.time()
logger.debug('function {} serialize time: {} secs'.format(
user_key, end - start))
if sync_to_shared_storage:
storage_key = create_mod_key(self.storage.prefix, user_key)
self.storage.put_module_dependencies(storage_key, func_module_str)
logger.debug(
'function and module dependencies has synced to shared storage, user_key: {}, storage_key: {}'
.format(user_key, storage_key))
def map(self, func, iterdata, extra_env=None, extra_meta=None,
invoke_pool_threads=64, data_all_as_one=True,
use_cached_runtime=True, overwrite_invoke_args=None, exclude_modules=[]):
"""
# FIXME work with an actual iterable instead of just a list
data_all_as_one : upload the data as a single object; fewer
tcp transactions (good) but potentially higher latency for workers (bad)
use_cached_runtime : if runtime has been cached, use that. When set
to False, redownloads runtime.
"""
data = list(iterdata)
if not data:
return []
host_job_meta = {}
pool = ThreadPool(invoke_pool_threads)
callset_id = wrenutil.create_callset_id()
### pickle func and all data (to capture module dependencies
func_and_data_ser, mod_paths = self.serializer([func] + data)
func_str = func_and_data_ser[0]
data_strs = func_and_data_ser[1:]
data_size_bytes = sum(len(x) for x in data_strs)
agg_data_key = None
host_job_meta['agg_data'] = False
host_job_meta['data_size_bytes'] = data_size_bytes
if data_size_bytes < wrenconfig.MAX_AGG_DATA_SIZE and data_all_as_one:
agg_data_key = storage_utils.create_agg_data_key(self.storage.prefix, callset_id)
agg_data_bytes, agg_data_ranges = self.agg_data(data_strs)
agg_upload_time = time.time()
self.storage.put_data(agg_data_key, agg_data_bytes)
host_job_meta['agg_data'] = True
host_job_meta['data_upload_time'] = time.time() - agg_upload_time
host_job_meta['data_upload_timestamp'] = time.time()
else:
# FIXME add warning that you wanted data all as one but
# it exceeded max data size
pass
for module in exclude_modules:
for mod_path in list(mod_paths):
if module in mod_path and mod_path in mod_paths:
mod_paths.remove(mod_path)
module_data = create_mod_data(mod_paths)
func_str_encoded = wrenutil.bytes_to_b64str(func_str)
#debug_foo = {'func' : func_str_encoded,
# 'module_data' : module_data}
#pickle.dump(debug_foo, open("/tmp/py35.debug.pickle", 'wb'))
### Create func and upload
func_module_str = json.dumps({'func' : func_str_encoded,
'module_data' : module_data})
host_job_meta['func_module_str_len'] = len(func_module_str)
func_upload_time = time.time()
func_key = create_func_key(self.storage.prefix, callset_id)
self.storage.put_func(func_key, func_module_str)
host_job_meta['func_upload_time'] = time.time() - func_upload_time
host_job_meta['func_upload_timestamp'] = time.time()
def invoke(data_str, callset_id, call_id, func_key,
host_job_meta,
agg_data_key=None, data_byte_range=None):
data_key, output_key, status_key \
= storage_utils.create_keys(self.storage.prefix, callset_id, call_id)
host_job_meta['job_invoke_timestamp'] = time.time()
if agg_data_key is None:
data_upload_time = time.time()
self.put_data(data_key, data_str,
callset_id, call_id)
data_upload_time = time.time() - data_upload_time
host_job_meta['data_upload_time'] = data_upload_time
host_job_meta['data_upload_timestamp'] = time.time()
data_key = data_key
else:
data_key = agg_data_key
return self.invoke_with_keys(func_key, data_key,
output_key,
status_key,
callset_id, call_id, extra_env,
extra_meta, data_byte_range,
use_cached_runtime, host_job_meta.copy(),
self.job_max_runtime,
overwrite_invoke_args=overwrite_invoke_args)
N = len(data)
call_result_objs = []
for i in range(N):
call_id = "{:05d}".format(i)
data_byte_range = None
if agg_data_key is not None:
data_byte_range = agg_data_ranges[i]
cb = pool.apply_async(invoke, (data_strs[i], callset_id,
call_id, func_key,
host_job_meta.copy(),
agg_data_key,
data_byte_range))
logger.info("map {} {} apply async".format(callset_id, call_id))
call_result_objs.append(cb)
res = [c.get() for c in call_result_objs]
pool.close()
pool.join()
logger.info("map invoked {} {} pool join".format(callset_id, call_id))
# FIXME take advantage of the callset to return a lot of these
# note these are just the invocation futures
return res
def map(self,
func,
iterdata,
extra_env=None,
extra_meta=None,
invoke_pool_threads=64,
data_all_as_one=True,
use_cached_runtime=True,
overwrite_invoke_args=None,
exclude_modules=None):
"""
:param func: the function to map over the data
:param iterdata: An iterable of input data
:param extra_env: Additional environment variables for lambda environment. Default None.
:param extra_meta: Additional metadata to pass to lambda. Default None.
:param invoke_pool_threads: Number of threads to use to invoke.
:param data_all_as_one: upload the data as a single object. Default True
:param use_cached_runtime: Use cached runtime whenever possible. Default true
:param overwrite_invoke_args: Overwrite other args. Mainly used for testing.
:param exclude_modules: Explicitly keep these modules from pickled dependencies.
:return: A list with size `len(iterdata)` of futures for each job
:rtype: list of futures.
Usage
>>> futures = pwex.map(foo, data_list)
"""
data = list(iterdata)
if not data:
return []
if self.map_item_limit is not None and len(data) > self.map_item_limit:
raise ValueError("len(data) ={}, exceeding map item limit of {}"\
"consider mapping over a smaller"\
"number of items".format(len(data),
self.map_item_limit))
host_job_meta = {}
pool = ThreadPool(invoke_pool_threads)
callset_id = wrenutil.create_callset_id()
### pickle func and all data (to capture module dependencies
func_and_data_ser, mod_paths = self.serializer([func] + data)
func_str = func_and_data_ser[0]
data_strs = func_and_data_ser[1:]
data_size_bytes = sum(len(x) for x in data_strs)
agg_data_key = None
host_job_meta['agg_data'] = False
host_job_meta['data_size_bytes'] = data_size_bytes
if data_size_bytes < wrenconfig.MAX_AGG_DATA_SIZE and data_all_as_one:
agg_data_key = storage_utils.create_agg_data_key(
self.storage.prefix, callset_id)
agg_data_bytes, agg_data_ranges = self.agg_data(data_strs)
agg_upload_time = time.time()
self.storage.put_data(agg_data_key, agg_data_bytes)
host_job_meta['agg_data'] = True
host_job_meta['data_upload_time'] = time.time() - agg_upload_time
host_job_meta['data_upload_timestamp'] = time.time()
else:
# FIXME add warning that you wanted data all as one but
# it exceeded max data size
pass
if exclude_modules:
for module in exclude_modules:
for mod_path in list(mod_paths):
if module in mod_path and mod_path in mod_paths:
mod_paths.remove(mod_path)
module_data = create_mod_data(mod_paths)
### Create func and upload
func_module_str = pickle.dumps(
{
'func': func_str,
'module_data': module_data
}, -1)
host_job_meta['func_module_str_len'] = len(func_module_str)
func_upload_time = time.time()
func_key = create_func_key(self.storage.prefix, callset_id)
self.storage.put_func(func_key, func_module_str)
host_job_meta['func_upload_time'] = time.time() - func_upload_time
host_job_meta['func_upload_timestamp'] = time.time()
def invoke(data_str,
callset_id,
call_id,
func_key,
host_job_meta,
agg_data_key=None,
data_byte_range=None):
data_key, output_key, status_key \
= storage_utils.create_keys(self.storage.prefix, callset_id, call_id)
host_job_meta['job_invoke_timestamp'] = time.time()
if agg_data_key is None:
data_upload_time = time.time()
self.put_data(data_key, data_str, callset_id, call_id)
data_upload_time = time.time() - data_upload_time
host_job_meta['data_upload_time'] = data_upload_time
host_job_meta['data_upload_timestamp'] = time.time()
data_key = data_key
else:
data_key = agg_data_key
return self.invoke_with_keys(
func_key,
data_key,
output_key,
status_key,
callset_id,
call_id,
extra_env,
extra_meta,
data_byte_range,
use_cached_runtime,
host_job_meta.copy(),
self.job_max_runtime,
overwrite_invoke_args=overwrite_invoke_args)
N = len(data)
call_result_objs = []
for i in range(N):
call_id = "{:05d}".format(i)
data_byte_range = None
if agg_data_key is not None:
data_byte_range = agg_data_ranges[i]
cb = pool.apply_async(
invoke, (data_strs[i], callset_id, call_id, func_key,
host_job_meta.copy(), agg_data_key, data_byte_range))
logger.info("map {} {} apply async".format(callset_id, call_id))
call_result_objs.append(cb)
res = [c.get() for c in call_result_objs]
pool.close()
pool.join()
logger.info("map invoked {} {} pool join".format(callset_id, call_id))
# FIXME take advantage of the callset to return a lot of these
# note these are just the invocation futures
return res