def test_utf8_str(self):
def foo(x):
return extmoduleutf8.unicode_str(x)
x = 1.0
fut = self.wrenexec.call_async(foo, x)
self.wrenexec.invoker.run_jobs()
res = fut.result()
self.assertEqual(res, extmoduleutf8.TEST_STR)
# class SerializeTest(unittest.TestCase):
# def test_simple(x):
# def func(x):
# return x + 1
# data = list(range(5))
# serializer = serialize.SerializeIndependent()
# func_and_data_ser, mod_paths = serializer([func] + data)
# for m in mod_paths:
# print(m)
config = pywren.wrenconfig.default()
info = runtime.get_runtime_info(config['runtime'])
print(info.keys())
for f in info['pkg_ver_list']:
print(f[0])
def __init__(self, aws_region, s3_bucket, s3_prefix, invoker,
runtime_s3_bucket, runtime_s3_key, job_max_runtime):
self.aws_region = aws_region
self.s3_bucket = s3_bucket
self.s3_prefix = s3_prefix
self.session = botocore.session.get_session()
self.invoker = invoker
self.s3client = self.session.create_client('s3',
region_name=aws_region)
self.job_max_runtime = job_max_runtime
self.runtime_bucket = runtime_s3_bucket
self.runtime_key = runtime_s3_key
self.runtime_meta_info = runtime.get_runtime_info(
runtime_s3_bucket, runtime_s3_key)
if not runtime.runtime_key_valid(self.runtime_meta_info):
raise Exception(
"The indicated runtime: s3://{}/{} is not approprite for this python version"
.format(runtime_s3_bucket, runtime_s3_key))
if 'preinstalls' in self.runtime_meta_info:
logger.info("using serializer with meta-supplied preinstalls")
self.serializer = serialize.SerializeIndependent(
self.runtime_meta_info['preinstalls'])
else:
self.serializer = serialize.SerializeIndependent()
def __init__(self, invoker, config, job_max_runtime):
self.invoker = invoker
self.job_max_runtime = job_max_runtime
self.config = config
self.storage_config = wrenconfig.extract_storage_config(self.config)
self.storage = storage.Storage(self.storage_config)
self.runtime_meta_info = runtime.get_runtime_info(config['runtime'])
# print('runtime_meta_info: ', self.runtime_meta_info)
self.runtime_meta_info['preinstalls'].append(['pandas', True])
self.runtime_meta_info['preinstalls'].append(['thrift', True])
self.runtime_meta_info['preinstalls'].append(['Thrift', True])
if 'preinstalls' in self.runtime_meta_info:
logger.info("using serializer with meta-supplied preinstalls")
self.serializer = serialize.SerializeIndependent(
self.runtime_meta_info['preinstalls'])
else:
self.serializer = serialize.SerializeIndependent()
self.map_item_limit = None
if 'scheduler' in self.config:
if 'map_item_limit' in config['scheduler']:
self.map_item_limit = config['scheduler']['map_item_limit']
def __init__(self, invoker, config, job_max_runtime):
self.invoker = invoker
self.job_max_runtime = job_max_runtime
self.config = config
self.storage_config = wrenconfig.extract_storage_config(self.config)
self.storage = storage.Storage(self.storage_config)
if self.config['storage_backend'] == 's3':
self.runtime_meta_info = runtime.get_runtime_info(config, self.storage)
elif self.config['storage_backend'] == 'google':
self.runtime_meta_info = runtime.get_runtime_info(config, self.storage)
if 'preinstalls' in self.runtime_meta_info:
logger.info("using serializer with meta-supplied preinstalls")
self.serializer = serialize.SerializeIndependent(self.runtime_meta_info['preinstalls'])
else:
self.serializer = serialize.SerializeIndependent()
def test_simple(x):
def func(x):
return x + 1
data = list(range(5))
serializer = serialize.SerializeIndependent()
func_and_data_ser, mod_paths = serializer([func] + data)
for m in mod_paths:
print(m)
config = pywren.wrenconfig.default()
runtime_bucket = config['runtime']['s3_bucket']
runtime_key = config['runtime']['s3_key']
info = runtime.get_runtime_info(runtime_bucket, runtime_key)
print(info.keys())
for f in info['pkg_ver_list']:
print(f[0])
def __init__(self, aws_region, s3_bucket, s3_prefix, invoker, config,
job_max_runtime):
self.aws_region = aws_region
self.s3_bucket = s3_bucket
self.s3_prefix = s3_prefix
self.config = config
self.session = botocore.session.get_session()
self.invoker = invoker
self.s3client = self.session.create_client('s3',
region_name=aws_region)
self.job_max_runtime = job_max_runtime
runtime_bucket = config['runtime']['s3_bucket']
runtime_key = config['runtime']['s3_key']
self.runtime_meta_info = runtime.get_runtime_info(
runtime_bucket, runtime_key)
if not runtime.runtime_key_valid(self.runtime_meta_info):
raise Exception(
"The indicated runtime: s3://{}/{} is not approprite for this python version"
.format(runtime_bucket, runtime_key))