def __init__(self, config, executor_id, internal_storage):
self.log_level = os.getenv('PYWREN_LOGLEVEL')
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_config = extract_compute_config(self.config)
self.is_pywren_function = is_pywren_function()
self.invokers = []
self.remote_invoker = self.config['pywren'].get('remote_invoker', False)
self.workers = self.config['pywren'].get('workers')
logger.debug('ExecutorID {} - Total available workers: {}'
.format(self.executor_id, self.workers))
self.compute_handlers = []
cb = self.compute_config['backend']
regions = self.compute_config[cb].get('region')
if regions and type(regions) == list:
for region in regions:
compute_config = self.compute_config.copy()
compute_config[cb]['region'] = region
self.compute_handlers.append(Compute(compute_config))
else:
self.compute_handlers.append(Compute(self.compute_config))
logger.debug('ExecutorID {} - Creating function invoker'.format(self.executor_id))
self.token_bucket_q = Queue()
self.pending_calls_q = Queue()
self.running_flag = Value('i', 0)
self.ongoing_activations = 0
self.job_monitor = JobMonitor(self.config, self.internal_storage, self.token_bucket_q)
def clean_all(config=None):
logger.info('Cleaning all PyWren information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean object storage temp dirs
sh = internal_storage.storage_handler
runtimes = sh.list_keys(storage_config['bucket'], RUNTIMES_PREFIX)
if runtimes:
sh.delete_objects(storage_config['bucket'], runtimes)
compute_handler.delete_all_runtimes()
clean_bucket(storage_config['bucket'],
JOBS_PREFIX,
internal_storage,
sleep=1)
# Clean local runtime_meta cache
if os.path.exists(CACHE_DIR):
shutil.rmtree(CACHE_DIR)
# Clean localhost temp dirs
localhost_jobs_path = os.path.join(TEMP, JOBS_PREFIX)
if os.path.exists(localhost_jobs_path):
shutil.rmtree(localhost_jobs_path)
localhost_runtimes_path = os.path.join(TEMP, RUNTIMES_PREFIX)
if os.path.exists(localhost_runtimes_path):
shutil.rmtree(localhost_runtimes_path)
def __init__(self, config, executor_id):
self.log_level = os.getenv('CB_LOG_LEVEL')
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
compute_config = extract_compute_config(config)
self.internal_compute = Compute(compute_config)
def __init__(self, config, log_level):
self.config = config
self.log_level = log_level
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(self.config)
self.remote_invoker = self.config['pywren'].get(
'remote_invoker', False)
self.rabbitmq_monitor = self.config['pywren'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
self.workers = self.config['pywren'].get('workers')
logger.debug('Total workers: {}'.format(self.workers))
self.compute_handlers = []
cb = compute_config['backend']
regions = compute_config[cb].get('region')
if regions and type(regions) == list:
for region in regions:
new_compute_config = compute_config.copy()
new_compute_config[cb]['region'] = region
self.compute_handlers.append(Compute(new_compute_config))
else:
self.compute_handlers.append(Compute(compute_config))
self.token_bucket_q = Queue()
self.pending_calls_q = Queue()
self.job_monitor = JobMonitor(self.config, self.internal_storage,
self.token_bucket_q)
def update_runtime(name, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
timeout = config['pywren']['runtime_timeout']
logger.info('Updating runtime: {}'.format(name))
if name != 'all':
runtime_meta = compute_handler.generate_runtime_meta(name)
else:
runtime_meta = None
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.create_runtime(runtime[0], runtime[1], timeout)
if runtime_meta:
try:
runtime_key = compute_handler.get_runtime_key(
runtime[0], runtime[1])
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise ("Unable to upload 'preinstalled modules' file into {}".
format(internal_storage.backend))
def clean_runtimes(config=None):
logger.info('Cleaning all runtimes and cache information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean local runtime_meta cache
if os.path.exists(CACHE_DIR):
shutil.rmtree(CACHE_DIR)
# Clean localhost dirs
localhost_jobs_path = os.path.join(TEMP, STORAGE_PREFIX_DEFAULT)
if os.path.exists(localhost_jobs_path):
shutil.rmtree(localhost_jobs_path)
localhost_runtimes_path = os.path.join(TEMP, RUNTIMES_PREFIX_DEFAULT)
if os.path.exists(localhost_runtimes_path):
shutil.rmtree(localhost_runtimes_path)
# Clean runtime metadata in the object storage
sh = internal_storage.storage_handler
runtimes = sh.list_keys(storage_config['bucket'], RUNTIMES_PREFIX_DEFAULT)
if runtimes:
sh.delete_objects(storage_config['bucket'], runtimes)
compute_handler.delete_all_runtimes()
def __init__(self, pywren_config):
self.config = pywren_config
self.rabbitmq_monitor = self.config['pywren'].get('rabbitmq_monitor', False)
self.store_status = strtobool(os.environ.get('STORE_STATUS', 'True'))
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.response = {'exception': False}
def __init__(self, config=None, runtime=None, runtime_memory=None, compute_backend=None,
compute_backend_region=None, log_level=None, rabbitmq_monitor=False):
"""
Initialize and return a ServerlessExecutor class.
:param config: Settings passed in here will override those in config file. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime
:param log_level: log level to use during the execution
:param rabbitmq_monitor: use rabbitmq as monitoring system
:return `ServerlessExecutor` object.
"""
self.start_time = time.time()
self._state = ExecutorState.new
self.config = default_config(config)
self.is_cf_cluster = is_cf_cluster()
self.data_cleaner = self.config['pywren']['data_cleaner']
# Overwrite runtime variables
if runtime:
self.config['pywren']['runtime'] = runtime
if runtime_memory:
self.config['pywren']['runtime_memory'] = int(runtime_memory)
if compute_backend:
self.config['pywren']['compute_backend'] = compute_backend
if compute_backend_region:
self.config['pywren']['compute_backend_region'] = compute_backend_region
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if(logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(logger.getEffectiveLevel())
if self.log_level:
os.environ["CB_LOG_LEVEL"] = self.log_level
if not self.is_cf_cluster:
default_logging_config(self.log_level)
if 'CB_EXECUTOR_ID' in os.environ:
self.executor_id = os.environ['CB_EXECUTOR_ID']
else:
self.executor_id = create_executor_id()
logger.debug('ServerlessExecutor created with ID: {}'.format(self.executor_id))
# RabbitMQ monitor configuration
self.rabbitmq_monitor = rabbitmq_monitor
if self.rabbitmq_monitor:
if self.config['rabbitmq']['amqp_url']:
os.environ["CB_RABBITMQ_MONITOR"] = 'True'
else:
self.rabbitmq_monitor = False
else:
self.config['rabbitmq']['amqp_url'] = None
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.invoker = Invoker(self.config, self.executor_id)
self.jobs = {}
def delete_runtime(name, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def load_config(config_data=None):
if config_data is None:
config_data = {}
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
config_data['pywren']['runtime'] = 'python' + \
version_str(sys.version_info)
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = MAX_CONCURRENT_WORKERS
if config_data['pywren']['runtime_memory'] not in RUNTIME_MEMORY_OPTIONS:
raise Exception('{} MB runtime is not available (Only {} MB)'.format(
config_data['pywren']['runtime_memory'], RUNTIME_MEMORY_OPTIONS))
if config_data['pywren']['runtime_memory'] > RUNTIME_MEMORY_MAX:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_MAX
if config_data['pywren']['runtime_timeout'] > RUNTIME_TIMEOUT_DEFAULT:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'gcp' not in config_data:
raise Exception("'gcp' section is mandatory in the configuration")
config_data['gcp']['retries'] = RETRIES
config_data['gcp']['retry_sleeps'] = RETRY_SLEEPS
# Put storage data into compute backend config dict entry
storage_config = dict()
storage_config['pywren'] = config_data['pywren'].copy()
storage_config['gcp_storage'] = config_data['gcp'].copy()
config_data['gcp']['storage'] = pw_config.extract_storage_config(
storage_config)
required_parameters_0 = ('project_name', 'service_account',
'credentials_path')
if not set(required_parameters_0) <= set(config_data['gcp']):
raise Exception(
"'project_name', 'service_account' and 'credentials_path' \
are mandatory under 'gcp' section")
if not exists(config_data['gcp']['credentials_path']) or not isfile(
config_data['gcp']['credentials_path']):
raise Exception("Path {} must be credentials JSON file.".format(
config_data['gcp']['credentials_path']))
config_data['gcp_functions'] = config_data['gcp'].copy()
if 'region' not in config_data['gcp_functions']:
config_data['gcp_functions']['region'] = config_data['pywren'][
'compute_backend_region']
def __init__(self, config, num_invokers, log_level):
self.config = config
self.num_invokers = num_invokers
self.log_level = log_level
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(self.config)
self.remote_invoker = self.config['pywren'].get(
'remote_invoker', False)
self.rabbitmq_monitor = self.config['pywren'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
self.num_workers = self.config['pywren'].get('workers')
logger.debug('Total workers: {}'.format(self.num_workers))
self.compute_handlers = []
cb = compute_config['backend']
regions = compute_config[cb].get('region')
if regions and type(regions) == list:
for region in regions:
new_compute_config = compute_config.copy()
new_compute_config[cb]['region'] = region
compute_handler = Compute(new_compute_config)
self.compute_handlers.append(compute_handler)
else:
if cb == 'localhost':
global CBH
if cb in CBH and CBH[
cb].compute_handler.num_workers != self.num_workers:
del CBH[cb]
if cb in CBH:
logger.info(
'{} compute handler already started'.format(cb))
compute_handler = CBH[cb]
self.compute_handlers.append(compute_handler)
else:
logger.info('Starting {} compute handler'.format(cb))
compute_handler = Compute(compute_config)
CBH[cb] = compute_handler
self.compute_handlers.append(compute_handler)
else:
compute_handler = Compute(compute_config)
self.compute_handlers.append(compute_handler)
self.token_bucket_q = Queue()
self.pending_calls_q = Queue()
self.job_monitor = JobMonitor(self.config, self.internal_storage,
self.token_bucket_q)
def __init__(self, config, executor_id, internal_storage):
self.log_level = os.getenv('PYWREN_LOGLEVEL')
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_config = extract_compute_config(self.config)
self.remote_invoker = self.config['pywren'].get(
'remote_invoker', False)
self.rabbitmq_monitor = self.config['pywren'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
self.workers = self.config['pywren'].get('workers')
logger.debug('ExecutorID {} - Total workers:'.format(self.workers))
self.compute_handlers = []
cb = self.compute_config['backend']
regions = self.compute_config[cb].get('region')
if regions and type(regions) == list:
for region in regions:
compute_config = self.compute_config.copy()
compute_config[cb]['region'] = region
self.compute_handlers.append(Compute(compute_config))
else:
self.compute_handlers.append(Compute(self.compute_config))
logger.debug('ExecutorID {} - Creating invoker process'.format(
self.executor_id))
self.token_bucket_q = Queue()
self.pending_calls_q = Queue()
self.invoker_process_stop_flag = Value('i', 0)
self.is_pywren_function = is_pywren_function()
if self.is_pywren_function or not is_unix_system():
self.invoker_process = Thread(target=self.run_process, args=())
else:
self.invoker_process = Process(target=self.run_process, args=())
self.invoker_process.daemon = True
self.invoker_process.start()
self.ongoing_activations = 0
def clean_runtimes(config=None):
logger.info('Cleaning all runtimes and cache information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean local runtime_meta cache
if os.path.exists(CACHE_DIR):
shutil.rmtree(CACHE_DIR)
sh = internal_storage.storage_handler
runtimes = sh.list_keys(storage_config['bucket'], 'runtime')
if runtimes:
sh.delete_objects(storage_config['bucket'], runtimes)
compute_handler.delete_all_runtimes()
def __init__(self, pywren_config, executor_id, internal_storage):
self.log_level = os.getenv('PYWREN_LOGLEVEL')
self.pywren_config = pywren_config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.pywren_config)
self.internal_storage = internal_storage
self.compute_config = extract_compute_config(self.pywren_config)
self.compute_handlers = []
cb = self.compute_config['backend']
regions = self.compute_config[cb].get('region')
if type(regions) == list:
for region in regions:
compute_config = self.compute_config.copy()
compute_config[cb]['region'] = region
self.compute_handlers.append(Compute(compute_config))
else:
self.compute_handlers.append(Compute(self.compute_config))
def create_runtime(name, memory=None, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
memory = config['pywren']['runtime_memory'] if not memory else memory
timeout = config['pywren']['runtime_timeout']
logger.info('Creating runtime: {}, memory: {}'.format(name, memory))
runtime_key = compute_handler.get_runtime_key(name, memory)
runtime_meta = compute_handler.create_runtime(name, memory, timeout=timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def clean_runtimes(config=None):
logger.info('Cleaning all runtimes')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean local runtime_meta cache
cache_dir = os.path.join(os.path.expanduser('~'), '.cloudbutton')
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
sh = internal_storage.storage_handler
runtimes = sh.list_keys(storage_config['bucket'], 'runtime')
if runtimes:
sh.delete_objects(storage_config['bucket'], runtimes)
compute_handler.delete_all_runtimes()
def run_tests(test_to_run, config=None):
global CONFIG, STORAGE_CONFIG, STORAGE
CONFIG = json.load(args.config) if config else default_config()
STORAGE_CONFIG = extract_storage_config(CONFIG)
STORAGE = InternalStorage(STORAGE_CONFIG).storage
suite = unittest.TestSuite()
if test_to_run == 'all':
suite.addTest(unittest.makeSuite(TestPywren))
else:
try:
suite.addTest(TestPywren(test_to_run))
except ValueError:
print("unknown test, use: --help")
sys.exit()
runner = unittest.TextTestRunner()
runner.run(suite)
def clean_all(config=None):
logger.info('Cleaning all PyWren information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean localhost executor temp dirs
shutil.rmtree(STORAGE_FOLDER, ignore_errors=True)
shutil.rmtree(DOCKER_FOLDER, ignore_errors=True)
# Clean object storage temp dirs
compute_handler.delete_all_runtimes()
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean local pywren cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def __init__(self, tr_config, result_queue):
super().__init__()
start_time = time.time()
self.config = tr_config
log_level = self.config['log_level']
self.result_queue = result_queue
cloud_logging_config(log_level)
self.stats = stats(self.config['stats_filename'])
self.stats.write('jobrunner_start', start_time)
cb_config = json.loads(os.environ.get('CB_CONFIG'))
self.storage_config = extract_storage_config(cb_config)
if 'SHOW_MEMORY_USAGE' in os.environ:
self.show_memory = eval(os.environ['SHOW_MEMORY_USAGE'])
else:
self.show_memory = False
self.func_key = self.config['func_key']
self.data_key = self.config['data_key']
self.data_byte_range = self.config['data_byte_range']
self.output_key = self.config['output_key']
def __init__(self, jr_config, jobrunner_conn):
start_time = time.time()
self.jr_config = jr_config
self.jobrunner_conn = jobrunner_conn
log_level = self.jr_config['log_level']
cloud_logging_config(log_level)
self.pywren_config = self.jr_config['pywren_config']
self.storage_config = extract_storage_config(self.pywren_config)
self.call_id = self.jr_config['call_id']
self.job_id = self.jr_config['job_id']
self.executor_id = self.jr_config['executor_id']
self.func_key = self.jr_config['func_key']
self.data_key = self.jr_config['data_key']
self.data_byte_range = self.jr_config['data_byte_range']
self.output_key = self.jr_config['output_key']
self.stats = stats(self.jr_config['stats_filename'])
self.stats.write('jobrunner_start', start_time)
self.show_memory = strtobool(
os.environ.get('SHOW_MEMORY_USAGE', 'False'))
def function_handler(event):
start_tstamp = time.time()
log_level = event['log_level']
cloud_logging_config(log_level)
logger.debug("Action handler started")
extra_env = event.get('extra_env', {})
os.environ.update(extra_env)
os.environ.update({'PYWREN_FUNCTION': 'True', 'PYTHONUNBUFFERED': 'True'})
config = event['config']
call_id = event['call_id']
job_id = event['job_id']
executor_id = event['executor_id']
exec_id = "{}/{}/{}".format(executor_id, job_id, call_id)
logger.info("Execution-ID: {}".format(exec_id))
runtime_name = event['runtime_name']
runtime_memory = event['runtime_memory']
execution_timeout = event['execution_timeout']
logger.debug("Runtime name: {}".format(runtime_name))
logger.debug("Runtime memory: {}MB".format(runtime_memory))
logger.debug("Function timeout: {}s".format(execution_timeout))
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
call_status = CallStatus(config, internal_storage)
call_status.response['host_submit_tstamp'] = event['host_submit_tstamp']
call_status.response['start_tstamp'] = start_tstamp
context_dict = {
'python_version': os.environ.get("PYTHON_VERSION"),
'call_id': call_id,
'job_id': job_id,
'executor_id': executor_id,
'activation_id': os.environ.get('__PW_ACTIVATION_ID')
}
call_status.response.update(context_dict)
show_memory_peak = strtobool(os.environ.get('SHOW_MEMORY_PEAK', 'False'))
try:
if version.__version__ != event['pywren_version']:
msg = (
"PyWren version mismatch. Host version: {} - Runtime version: {}"
.format(event['pywren_version'], version.__version__))
raise RuntimeError('HANDLER', msg)
# send init status event
call_status.send('__init__')
# call_status.response['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {
'PYWREN_CONFIG': json.dumps(config),
'PYWREN_EXECUTION_ID': exec_id,
'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH)
}
os.environ.update(custom_env)
jobrunner_stats_dir = os.path.join(STORAGE_FOLDER,
storage_config['bucket'],
JOBS_PREFIX, executor_id, job_id,
call_id)
os.makedirs(jobrunner_stats_dir, exist_ok=True)
jobrunner_stats_filename = os.path.join(jobrunner_stats_dir,
'jobrunner.stats.txt')
jobrunner_config = {
'pywren_config':
config,
'call_id':
call_id,
'job_id':
job_id,
'executor_id':
executor_id,
'func_key':
func_key,
'data_key':
data_key,
'log_level':
log_level,
'data_byte_range':
data_byte_range,
'output_key':
create_output_key(JOBS_PREFIX, executor_id, job_id, call_id),
'stats_filename':
jobrunner_stats_filename
}
if show_memory_peak:
mm_handler_conn, mm_conn = Pipe()
memory_monitor = Thread(target=memory_monitor_worker,
args=(mm_conn, ))
memory_monitor.start()
handler_conn, jobrunner_conn = Pipe()
jobrunner = JobRunner(jobrunner_config, jobrunner_conn,
internal_storage)
logger.debug('Starting JobRunner process')
local_execution = strtobool(
os.environ.get('__PW_LOCAL_EXECUTION', 'False'))
jrp = Thread(target=jobrunner.run) if local_execution else Process(
target=jobrunner.run)
jrp.start()
jrp.join(execution_timeout)
logger.debug('JobRunner process finished')
if jrp.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
try:
jrp.terminate()
except Exception:
# thread does not have terminate method
pass
msg = ('Function exceeded maximum time of {} seconds and was '
'killed'.format(execution_timeout))
raise TimeoutError('HANDLER', msg)
if show_memory_peak:
mm_handler_conn.send('STOP')
memory_monitor.join()
peak_memory_usage = int(mm_handler_conn.recv())
logger.info("Peak memory usage: {}".format(
sizeof_fmt(peak_memory_usage)))
call_status.response['peak_memory_usage'] = peak_memory_usage
if not handler_conn.poll():
logger.error(
'No completion message received from JobRunner process')
logger.debug('Assuming memory overflow...')
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Function exceeded maximum memory and was killed'
raise MemoryError('HANDLER', msg)
if os.path.exists(jobrunner_stats_filename):
with open(jobrunner_stats_filename, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
call_status.response[key] = float(value)
except Exception:
call_status.response[key] = value
if key in [
'exception', 'exc_pickle_fail', 'result',
'new_futures'
]:
call_status.response[key] = eval(value)
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------',
flush=True)
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------',
flush=True)
call_status.response['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
call_status.response['exc_info'] = str(pickled_exc)
finally:
call_status.response['end_tstamp'] = time.time()
call_status.send('__end__')
for key in extra_env:
os.environ.pop(key)
logger.info("Finished")
def __init__(self,
config=None,
runtime=None,
runtime_memory=None,
compute_backend=None,
compute_backend_region=None,
storage_backend=None,
storage_backend_region=None,
rabbitmq_monitor=None,
log_level=None):
"""
Initialize a FunctionExecutor class.
:param config: Settings passed in here will override those in config file. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime. Default None.
:param compute_backend: Name of the compute backend to use. Default None.
:param compute_backend_region: Name of the compute backend region to use. Default None.
:param storage_backend: Name of the storage backend to use. Default None.
:param storage_backend_region: Name of the storage backend region to use. Default None.
:param log_level: log level to use during the execution. Default None.
:param rabbitmq_monitor: use rabbitmq as the monitoring system. Default None.
:return `FunctionExecutor` object.
"""
self.start_time = time.time()
self._state = FunctionExecutor.State.New
self.is_remote_cluster = is_remote_cluster()
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if (logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(
logger.getEffectiveLevel())
if self.log_level:
os.environ["PYWREN_LOGLEVEL"] = self.log_level
if not self.is_remote_cluster:
default_logging_config(self.log_level)
# Overwrite pywren config parameters
config_ow = {'pywren': {}}
if runtime is not None:
config_ow['pywren']['runtime'] = runtime
if runtime_memory is not None:
config_ow['pywren']['runtime_memory'] = int(runtime_memory)
if compute_backend is not None:
config_ow['pywren']['compute_backend'] = compute_backend
if compute_backend_region is not None:
config_ow['pywren'][
'compute_backend_region'] = compute_backend_region
if storage_backend is not None:
config_ow['pywren']['storage_backend'] = storage_backend
if storage_backend_region is not None:
config_ow['pywren'][
'storage_backend_region'] = storage_backend_region
if rabbitmq_monitor is not None:
config_ow['pywren']['rabbitmq_monitor'] = rabbitmq_monitor
self.config = default_config(config, config_ow)
self.executor_id = create_executor_id()
logger.debug('FunctionExecutor created with ID: {}'.format(
self.executor_id))
# RabbitMQ monitor configuration
self.rabbitmq_monitor = self.config['pywren'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config[
'rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception(
"You cannot use rabbitmq_mnonitor since 'amqp_url'"
" is not present in configuration")
self.data_cleaner = self.config['pywren']['data_cleaner']
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.invoker = FunctionInvoker(self.config, self.executor_id,
self.internal_storage)
self.jobs = {}
def function_handler(event):
start_time = time.time()
log_level = event['log_level']
cloud_logging_config(log_level)
logger.debug("Action handler started")
response_status = {'exception': False}
response_status['host_submit_time'] = event['host_submit_time']
response_status['start_time'] = start_time
context_dict = {
'python_version': os.environ.get("PYTHON_VERSION"),
}
config = event['config']
storage_config = extract_storage_config(config)
call_id = event['call_id']
job_id = event['job_id']
executor_id = event['executor_id']
logger.info("Execution ID: {}/{}/{}".format(executor_id, job_id, call_id))
execution_timeout = event['execution_timeout']
logger.debug(
"Set function execution timeout to {}s".format(execution_timeout))
status_key = event['status_key']
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
output_key = event['output_key']
extra_env = event.get('extra_env', {})
response_status['call_id'] = call_id
response_status['job_id'] = job_id
response_status['executor_id'] = executor_id
# response_status['func_key'] = func_key
# response_status['data_key'] = data_key
# response_status['output_key'] = output_key
# response_status['status_key'] = status_key
try:
if version.__version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "PyWren version mismatch",
version.__version__, event['pywren_version'])
# response_status['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {
'PYWREN_CONFIG': json.dumps(config),
'PYWREN_REMOTE': 'TRUE',
'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH),
'PYTHONUNBUFFERED': 'True'
}
os.environ.update(custom_env)
os.environ.update(extra_env)
jobrunner_config = {
'pywren_config': config,
'call_id': call_id,
'job_id': job_id,
'executor_id': executor_id,
'func_key': func_key,
'data_key': data_key,
'log_level': log_level,
'data_byte_range': data_byte_range,
'python_module_path': PYTHON_MODULE_PATH,
'output_key': output_key,
'stats_filename': JOBRUNNER_STATS_FILENAME
}
if os.path.exists(JOBRUNNER_STATS_FILENAME):
os.remove(JOBRUNNER_STATS_FILENAME)
setup_time = time.time()
response_status['setup_time'] = round(setup_time - start_time, 8)
result_queue = multiprocessing.Queue()
tr = JobRunner(jobrunner_config, result_queue)
tr.daemon = True
logger.debug('Starting JobRunner process')
tr.start()
tr.join(execution_timeout)
logger.debug('Finished JobRunner process')
response_status['exec_time'] = round(time.time() - setup_time, 8)
if tr.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
tr.terminate()
msg = ('Jobrunner process exceeded maximum time of {} '
'seconds and was killed'.format(execution_timeout))
raise Exception('OUTATIME', msg)
if result_queue.empty():
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Jobrunner process exceeded maximum memory and was killed'
raise Exception('OUTOFMEMORY', msg)
# print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True))
# print(subprocess.check_output("find {}".format(os.getcwd()), shell=True))
if os.path.exists(JOBRUNNER_STATS_FILENAME):
with open(JOBRUNNER_STATS_FILENAME, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
response_status[key] = float(value)
except Exception:
response_status[key] = value
if key in [
'exception', 'exc_pickle_fail', 'result',
'new_futures'
]:
response_status[key] = eval(value)
# response_status['server_info'] = get_server_info()
response_status.update(context_dict)
response_status['end_time'] = time.time()
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------',
flush=True)
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------',
flush=True)
response_status['end_time'] = time.time()
response_status['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
response_status['exc_info'] = str(pickled_exc)
finally:
store_status = strtobool(os.environ.get('STORE_STATUS', 'True'))
dmpd_response_status = json.dumps(response_status)
drs = sizeof_fmt(len(dmpd_response_status))
rabbitmq_monitor = config['pywren'].get('rabbitmq_monitor', False)
if rabbitmq_monitor and store_status:
rabbit_amqp_url = config['rabbitmq'].get('amqp_url')
status_sent = False
output_query_count = 0
params = pika.URLParameters(rabbit_amqp_url)
queue = '{}-{}'.format(executor_id, job_id)
while not status_sent and output_query_count < 5:
output_query_count = output_query_count + 1
try:
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue=queue, auto_delete=True)
channel.basic_publish(exchange='',
routing_key=queue,
body=dmpd_response_status)
connection.close()
logger.info(
"Execution status sent to rabbitmq - Size: {}".format(
drs))
status_sent = True
except Exception as e:
logger.error("Unable to send status to rabbitmq")
logger.error(str(e))
logger.info('Retrying to send status to rabbitmq...')
time.sleep(0.2)
if store_status:
internal_storage = InternalStorage(storage_config)
logger.info(
"Storing execution stats - status.json - Size: {}".format(drs))
internal_storage.put_data(status_key, dmpd_response_status)
import pywren_ibm_cloud as pywren
import urllib.request
from pywren_ibm_cloud.storage import InternalStorage
from pywren_ibm_cloud.config import default_config, extract_storage_config
from multiprocessing.pool import ThreadPool
import logging
# logging.basicConfig(level=logging.DEBUG)
parser = argparse.ArgumentParser(description="test all PyWren's functionality", usage='python -m pywren_ibm_cloud.tests [-c CONFIG] [-f TESTNAME]')
parser.add_argument('-c', '--config', type=argparse.FileType('r'), metavar='', default=None, help="use json config file")
parser.add_argument('-t', '--test', metavar='', default='all', help='run a specific test, type "-t help" for tests list')
args = parser.parse_args()
CONFIG = default_config()
STORAGE_CONFIG = extract_storage_config(CONFIG)
STORAGE = InternalStorage(STORAGE_CONFIG).storage_handler
PREFIX = '__pywren.test'
TEST_FILES_URLS = ["http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.enron.txt",
"http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.kos.txt",
"http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.nips.txt",
"http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.nytimes.txt",
"http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.pubmed.txt"]
def initTests():
print('Uploading test files...')
def up(param):
i, url = param
content = urllib.request.urlopen(url).read()
def function_handler(event):
start_time = time.time()
logger.debug("Action handler started")
response_status = {'exception': False}
response_status['host_submit_time'] = event['host_submit_time']
response_status['start_time'] = start_time
context_dict = {
'ibm_cf_request_id': os.environ.get("__OW_ACTIVATION_ID"),
'ibm_cf_python_version': os.environ.get("PYTHON_VERSION"),
}
config = event['config']
storage_config = extract_storage_config(config)
log_level = event['log_level']
cloud_logging_config(log_level)
call_id = event['call_id']
job_id = event['job_id']
executor_id = event['executor_id']
logger.info("Execution ID: {}/{}/{}".format(executor_id, job_id, call_id))
task_execution_timeout = event.get("task_execution_timeout",
590) # default for CF
status_key = event['status_key']
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
output_key = event['output_key']
extra_env = event.get('extra_env', {})
response_status['call_id'] = call_id
response_status['job_id'] = job_id
response_status['executor_id'] = executor_id
# response_status['func_key'] = func_key
# response_status['data_key'] = data_key
# response_status['output_key'] = output_key
# response_status['status_key'] = status_key
try:
if version.__version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "PyWren version mismatch",
version.__version__, event['pywren_version'])
# response_status['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {
'CB_CONFIG': json.dumps(config),
'CB_CALL_ID': call_id,
'CB_JOB_ID': job_id,
'CB_EXECUTOR_ID': executor_id,
'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH),
'PYTHONUNBUFFERED': 'True'
}
os.environ.update(custom_env)
os.environ.update(extra_env)
# pass a full json blob
jobrunner_config = {
'func_key': func_key,
'data_key': data_key,
'log_level': log_level,
'data_byte_range': data_byte_range,
'python_module_path': PYTHON_MODULE_PATH,
'output_key': output_key,
'stats_filename': JOBRUNNER_STATS_FILENAME
}
if os.path.exists(JOBRUNNER_STATS_FILENAME):
os.remove(JOBRUNNER_STATS_FILENAME)
setup_time = time.time()
response_status['setup_time'] = round(setup_time - start_time, 8)
result_queue = multiprocessing.Queue()
tr = JobRunner(jobrunner_config, result_queue)
tr.daemon = True
logger.info("Starting JobRunner process")
tr.start()
tr.join(task_execution_timeout)
response_status['exec_time'] = round(time.time() - setup_time, 8)
if tr.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
logger.error(
"Process exceeded maximum runtime of {} seconds".format(
task_execution_timeout))
# Send the signal to all the process groups
tr.terminate()
raise Exception("OUTATIME",
"Process executed for too long and was killed")
try:
# Only 1 message is returned by jobrunner
result_queue.get(block=False)
except Exception:
# If no message, this means that the process was killed due an exception pickling an exception
raise Exception(
"EXCPICKLEERROR",
"PyWren was unable to pickle the exception, check function logs"
)
# print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True))
# print(subprocess.check_output("find {}".format(os.getcwd()), shell=True))
if os.path.exists(JOBRUNNER_STATS_FILENAME):
with open(JOBRUNNER_STATS_FILENAME, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
response_status[key] = float(value)
except Exception:
response_status[key] = value
if key == 'exception' or key == 'exc_pickle_fail' \
or key == 'result':
response_status[key] = eval(value)
# response_status['server_info'] = get_server_info()
response_status.update(context_dict)
response_status['end_time'] = time.time()
except Exception as e:
# internal runtime exceptions
logger.error("There was an exception: {}".format(str(e)))
response_status['end_time'] = time.time()
response_status['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
response_status['exc_info'] = str(pickled_exc)
finally:
store_status = strtobool(os.environ.get('STORE_STATUS', 'True'))
rabbit_amqp_url = config['rabbitmq'].get('amqp_url')
dmpd_response_status = json.dumps(response_status)
drs = sizeof_fmt(len(dmpd_response_status))
if rabbit_amqp_url and store_status:
status_sent = False
output_query_count = 0
while not status_sent and output_query_count < 5:
output_query_count = output_query_count + 1
try:
params = pika.URLParameters(rabbit_amqp_url)
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue=executor_id, auto_delete=True)
channel.basic_publish(exchange='',
routing_key=executor_id,
body=dmpd_response_status)
connection.close()
logger.info(
"Execution stats sent to rabbitmq - Size: {}".format(
drs))
status_sent = True
except Exception as e:
logger.error("Unable to send status to rabbitmq")
logger.error(str(e))
logger.info('Retrying to send stats to rabbitmq...')
time.sleep(0.2)
if store_status:
internal_storage = InternalStorage(storage_config)
logger.info(
"Storing execution stats - status.json - Size: {}".format(drs))
internal_storage.put_data(status_key, dmpd_response_status)
from concurrent.futures.thread import ThreadPoolExecutor
CENTROIDS_BUCKET = 'omeruseast'
MOL_DB_PREFIX = 'metabolomics/db/centroids_chunks'
CENTROIDS_SEGMENTS_PREFIX = 'metabolomics/vm_db_segments'
CENTR_SEGM_PATH = '/data/metabolomics/db/segms'
MZ_MIN = 79.99708557
MZ_MAX = 499.97909546
DS_SEGMENTS_N = 15
DS_SEGM_SIZE_MB = 100
from pywren_ibm_cloud.storage import InternalStorage
from pywren_ibm_cloud.config import default_config, extract_storage_config
PYWREN_CONFIG = default_config()
STORAGE_CONFIG = extract_storage_config(PYWREN_CONFIG)
STORAGE = InternalStorage(STORAGE_CONFIG).storage_handler
def download_database(storage, bucket, prefix):
keys = storage.list_keys(bucket, prefix)
def _download(key):
data_stream = storage.get_object(bucket, key, stream=True)
return pd.read_msgpack(data_stream).sort_values('mz')
with ThreadPoolExecutor() as pool:
centroids_df = pd.concat(list(pool.map(_download, keys)))
return centroids_df
def __init__(self, config, executor_id, compute_handler):
self.log_level = os.getenv('CB_LOG_LEVEL')
self.config = config
self.executor_id = executor_id
self.compute = compute_handler
self.storage_config = extract_storage_config(self.config)
