def clean_all(config=None):
logger.info('Cleaning all PyWren information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean object storage temp dirs
sh = internal_storage.storage_handler
runtimes = sh.list_keys(storage_config['bucket'], RUNTIMES_PREFIX)
if runtimes:
sh.delete_objects(storage_config['bucket'], runtimes)
compute_handler.delete_all_runtimes()
clean_os_bucket(storage_config['bucket'],
JOBS_PREFIX,
internal_storage,
sleep=1)
# Clean local runtime_meta cache
if os.path.exists(CACHE_DIR):
shutil.rmtree(CACHE_DIR)
# Clean localhost temp dirs
localhost_jobs_path = os.path.join(TEMP, JOBS_PREFIX)
if os.path.exists(localhost_jobs_path):
shutil.rmtree(localhost_jobs_path)
localhost_runtimes_path = os.path.join(TEMP, RUNTIMES_PREFIX)
if os.path.exists(localhost_runtimes_path):
shutil.rmtree(localhost_runtimes_path)
def build_runtime(name, file, config=None):
config = default_config(config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
compute_handler.build_runtime(name, file)
create_runtime(name, config=config)
update_runtime(name, config=config)
def delete_runtime(name, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def clean_runtimes(config=None):
logger.info('Cleaning all runtimes and cache information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean local runtime_meta cache
if os.path.exists(CACHE_DIR):
shutil.rmtree(CACHE_DIR)
sh = internal_storage.storage_handler
runtimes = sh.list_keys(storage_config['bucket'], 'runtime')
if runtimes:
sh.delete_objects(storage_config['bucket'], runtimes)
compute_handler.delete_all_runtimes()
def create_runtime(name, memory=None, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
memory = config['pywren']['runtime_memory'] if not memory else memory
timeout = config['pywren']['runtime_timeout']
logger.info('Creating runtime: {}, memory: {}'.format(name, memory))
runtime_key = compute_handler.get_runtime_key(name, memory)
runtime_meta = compute_handler.create_runtime(name, memory, timeout=timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def update_runtime(name, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
timeout = config['pywren']['runtime_timeout']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def __init__(self, config = None, runtime=None, runtime_memory=None, compute_backend=None,
compute_backend_region=None, storage_backend=None, storage_backend_region=None,
workers=None, rabbitmq_monitor=None, remote_invoker=None, log_level=None):
self.start_time = time.time()
self._state = self.State.New
self.is_pywren_function = is_pywren_function()
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if(logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(logger.getEffectiveLevel())
if self.log_level:
os.environ["PYWREN_LOGLEVEL"] = self.log_level
if not self.is_pywren_function:
default_logging_config(self.log_level)
# Overwrite pywren config parameters
pw_config_ow = {}
if runtime is not None:
pw_config_ow['runtime'] = runtime
if runtime_memory is not None:
pw_config_ow['runtime_memory'] = runtime_memory
if compute_backend is not None:
pw_config_ow['compute_backend'] = compute_backend
if compute_backend_region is not None:
pw_config_ow['compute_backend_region'] = compute_backend_region
if storage_backend is not None:
pw_config_ow['storage_backend'] = storage_backend
if storage_backend_region is not None:
pw_config_ow['storage_backend_region'] = storage_backend_region
if workers is not None:
pw_config_ow['workers'] = workers
if rabbitmq_monitor is not None:
pw_config_ow['rabbitmq_monitor'] = rabbitmq_monitor
if remote_invoker is not None:
pw_config_ow['remote_invoker'] = remote_invoker
self.config = default_config(copy.deepcopy(config), pw_config_ow)
self.executor_id = create_executor_id()
logger.debug('FunctionExecutor created with ID: {}'.format(self.executor_id))
self.data_cleaner = self.config['pywren'].get('data_cleaner', True)
self.rabbitmq_monitor = self.config['pywren'].get('rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception("You cannot use rabbitmq_mnonitor since 'amqp_url'"
" is not present in configuration")
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.invoker = FunctionInvoker(self.config, self.executor_id, self.internal_storage)
self.futures = []
self.total_jobs = 0
self.cleaned_jobs = set()