def clean_all(config=None):
logger.info('Cleaning all PyWren information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean object storage temp dirs
sh = internal_storage.storage_handler
runtimes = sh.list_keys(storage_config['bucket'], RUNTIMES_PREFIX)
if runtimes:
sh.delete_objects(storage_config['bucket'], runtimes)
compute_handler.delete_all_runtimes()
clean_os_bucket(storage_config['bucket'],
JOBS_PREFIX,
internal_storage,
sleep=1)
# Clean local runtime_meta cache
if os.path.exists(CACHE_DIR):
shutil.rmtree(CACHE_DIR)
# Clean localhost temp dirs
localhost_jobs_path = os.path.join(TEMP, JOBS_PREFIX)
if os.path.exists(localhost_jobs_path):
shutil.rmtree(localhost_jobs_path)
localhost_runtimes_path = os.path.join(TEMP, RUNTIMES_PREFIX)
if os.path.exists(localhost_runtimes_path):
shutil.rmtree(localhost_runtimes_path)
def __init__(self, config, executor_id, internal_storage):
self.log_level = os.getenv('PYWREN_LOGLEVEL')
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_config = extract_compute_config(self.config)
self.is_pywren_function = is_pywren_function()
self.remote_invoker = self.config['pywren'].get('remote_invoker', False)
self.workers = self.config['pywren'].get('workers')
logger.debug('ExecutorID {} - Total workers: {}'.format(self.executor_id, self.workers))
self.compute_handlers = []
cb = self.compute_config['backend']
regions = self.compute_config[cb].get('region')
if regions and type(regions) == list:
for region in regions:
compute_config = self.compute_config.copy()
compute_config[cb]['region'] = region
self.compute_handlers.append(Compute(compute_config))
else:
self.compute_handlers.append(Compute(self.compute_config))
logger.debug('ExecutorID {} - Creating function invoker'.format(self.executor_id))
self.token_bucket_q = Queue()
self.pending_calls_q = Queue()
self.running_flag = Value('i', 0)
self.ongoing_activations = 0
self.job_monitor = JobMonitor(self.config, self.internal_storage, self.token_bucket_q)
def __init__(self, pywren_config):
self.config = pywren_config
self.rabbitmq_monitor = self.config['pywren'].get(
'rabbitmq_monitor', False)
self.store_status = strtobool(os.environ.get('STORE_STATUS', 'True'))
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.response = {'exception': False}
def delete_runtime(name, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def clean_runtimes(config=None):
logger.info('Cleaning all runtimes and cache information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean local runtime_meta cache
if os.path.exists(CACHE_DIR):
shutil.rmtree(CACHE_DIR)
sh = internal_storage.storage_handler
runtimes = sh.list_keys(storage_config['bucket'], 'runtime')
if runtimes:
sh.delete_objects(storage_config['bucket'], runtimes)
compute_handler.delete_all_runtimes()
def create_runtime(name, memory=None, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
memory = config['pywren']['runtime_memory'] if not memory else memory
timeout = config['pywren']['runtime_timeout']
logger.info('Creating runtime: {}, memory: {}'.format(name, memory))
runtime_key = compute_handler.get_runtime_key(name, memory)
runtime_meta = compute_handler.create_runtime(name, memory, timeout=timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def update_runtime(name, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
timeout = config['pywren']['runtime_timeout']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def __init__(self, jr_config, jobrunner_conn):
start_time = time.time()
self.jr_config = jr_config
self.jobrunner_conn = jobrunner_conn
log_level = self.jr_config['log_level']
cloud_logging_config(log_level)
self.pywren_config = self.jr_config['pywren_config']
self.storage_config = extract_storage_config(self.pywren_config)
self.call_id = self.jr_config['call_id']
self.job_id = self.jr_config['job_id']
self.executor_id = self.jr_config['executor_id']
self.func_key = self.jr_config['func_key']
self.data_key = self.jr_config['data_key']
self.data_byte_range = self.jr_config['data_byte_range']
self.output_key = self.jr_config['output_key']
self.stats = stats(self.jr_config['stats_filename'])
self.stats.write('jobrunner_start', start_time)
self.show_memory = strtobool(
os.environ.get('SHOW_MEMORY_USAGE', 'False'))
def __init__(self, config = None, runtime=None, runtime_memory=None, compute_backend=None,
compute_backend_region=None, storage_backend=None, storage_backend_region=None,
workers=None, rabbitmq_monitor=None, remote_invoker=None, log_level=None):
self.start_time = time.time()
self._state = self.State.New
self.is_pywren_function = is_pywren_function()
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if(logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(logger.getEffectiveLevel())
if self.log_level:
os.environ["PYWREN_LOGLEVEL"] = self.log_level
if not self.is_pywren_function:
default_logging_config(self.log_level)
# Overwrite pywren config parameters
pw_config_ow = {}
if runtime is not None:
pw_config_ow['runtime'] = runtime
if runtime_memory is not None:
pw_config_ow['runtime_memory'] = runtime_memory
if compute_backend is not None:
pw_config_ow['compute_backend'] = compute_backend
if compute_backend_region is not None:
pw_config_ow['compute_backend_region'] = compute_backend_region
if storage_backend is not None:
pw_config_ow['storage_backend'] = storage_backend
if storage_backend_region is not None:
pw_config_ow['storage_backend_region'] = storage_backend_region
if workers is not None:
pw_config_ow['workers'] = workers
if rabbitmq_monitor is not None:
pw_config_ow['rabbitmq_monitor'] = rabbitmq_monitor
if remote_invoker is not None:
pw_config_ow['remote_invoker'] = remote_invoker
self.config = default_config(copy.deepcopy(config), pw_config_ow)
self.executor_id = create_executor_id()
logger.debug('FunctionExecutor created with ID: {}'.format(self.executor_id))
self.data_cleaner = self.config['pywren'].get('data_cleaner', True)
self.rabbitmq_monitor = self.config['pywren'].get('rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception("You cannot use rabbitmq_mnonitor since 'amqp_url'"
" is not present in configuration")
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.invoker = FunctionInvoker(self.config, self.executor_id, self.internal_storage)
self.futures = []
self.total_jobs = 0
self.cleaned_jobs = set()