def _run_job(self, job): """ Run a job """ if self.customized_runtime: logger.debug( 'ExecutorID {} | JobID {} - Customized runtime activated'. format(job.executor_id, job.job_id)) job.runtime_name = self.runtime_name extend_runtime(job, self.compute_handler, self.internal_storage) self.runtime_name = job.runtime_name logger.info('ExecutorID {} | JobID {} - Starting function ' 'invocation: {}() - Total: {} activations'.format( job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.debug('ExecutorID {} | JobID {} - Chunksize:' ' {} - Worker processes: {}'.format( job.executor_id, job.job_id, job.chunksize, job.worker_processes)) self.prometheus.send_metric(name='job_total_calls', value=job.total_calls, type='counter', labels=(('job_id_', job.job_key), ('function_name', job.function_name))) self.prometheus.send_metric(name='job_runtime_memory', value=job.runtime_memory or 0, type='counter', labels=(('job_id_', job.job_key), ('function_name', job.function_name))) try: job.runtime_name = self.runtime_name self._invoke_job(job) except (KeyboardInterrupt, Exception) as e: self.stop() raise e log_file = os.path.join(LOGS_DIR, job.job_key + '.log') logger.info( "ExecutorID {} | JobID {} - View execution logs at {}".format( job.executor_id, job.job_id, log_file)) # Create all futures futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) job.futures = futures return futures
def run(self, job_description): """ Run a job """ job_description['runtime_name'] = self.runtime_name job_description['runtime_memory'] = None job_description['runtime_timeout'] = None job = SimpleNamespace(**job_description) payload = { 'config': self.config, 'log_level': logging.getLevelName(logger.getEffectiveLevel()), 'executor_id': job.executor_id, 'job_id': job.job_id, 'job_description': job_description, 'lithops_version': __version__ } self.compute_handler.run_job(payload) log_msg = ('ExecutorID {} | JobID {} - Invocation done'.format( job.executor_id, job.job_id)) logger.info(log_msg) futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job_description, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures
def run(self, job): """ Run a job """ job.runtime_name = self.runtime_name payload = { 'config': self.config, 'log_level': logging.getLevelName(logger.getEffectiveLevel()), 'executor_id': job.executor_id, 'job_id': job.job_id, 'job_description': job.__dict__, 'lithops_version': __version__ } self.compute_handler.run_job(payload) log_msg = ('ExecutorID {} | JobID {} - {}() Invocation done'.format( job.executor_id, job.job_id, job.function_name)) logger.info(log_msg) if not self.log_active: print(log_msg) futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures
def run(self, job): """ Run a job """ logger.info('ExecutorID {} | JobID {} - Starting function' ' invocation: {}() - Total: {} activations'.format( job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.debug( 'ExecutorID {} | JobID {} - Chunksize: {} - Worker processes: {}'. format(job.executor_id, job.job_id, job.chunksize, job.worker_processes)) job.runtime_name = self.runtime_name self.prometheus.send_metric(name='job_total_calls', value=job.total_calls, labels=(('executor_id', job.executor_id), ('job_id', job.job_id), ('function_name', job.function_name))) payload = self._create_payload(job) payload['call_ids'] = [ "{:05d}".format(i) for i in range(job.total_calls) ] log_file = os.path.join(LOGS_DIR, payload['job_key'] + '.log') logger.info( "ExecutorID {} | JobID {} - View execution logs at {}".format( job.executor_id, job.job_id, log_file)) self.compute_handler.run_job(payload) futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures
def run(self, job): """ Run a job """ job.runtime_name = self.runtime_name self.prometheus.send_metric(name='job_total_calls', value=job.total_calls, labels=( ('job_id', job.job_id), ('function_name', job.function_name) )) payload = {'config': self.config, 'log_level': self.log_level, 'executor_id': job.executor_id, 'job_id': job.job_id, 'job_description': job.__dict__, 'lithops_version': __version__} self.compute_handler.run_job(payload) log_msg = ('ExecutorID {} | JobID {} - {}() Invocation done - Total: {} activations' .format(job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) if not self.log_active: print(log_msg) futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures
def run(self, job): """ Run a job described in job_description """ job.runtime_name = self.runtime_name try: while True: self.token_bucket_q.get_nowait() self.ongoing_activations -= 1 except Exception: pass self.prometheus.send_metric(name='job_total_calls', value=job.total_calls, labels=( ('job_id', job.job_id), ('function_name', job.function_name) )) if self.remote_invoker: """ Remote Invocation Use a single cloud function to perform all the function invocations """ old_stdout = sys.stdout sys.stdout = open(os.devnull, 'w') self.select_runtime(job.job_id, self.REMOTE_INVOKER_MEMORY) sys.stdout = old_stdout log_msg = ('ExecutorID {} | JobID {} - Starting remote function ' 'invocation: {}() - Total: {} activations' .format(job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) if not self.log_active: print(log_msg) th = Thread(target=self._invoke_remote, args=(job,), daemon=True) th.start() time.sleep(0.1) else: """ Normal Invocation Use local threads to perform all the function invocations """ try: if self.running_flag.value == 0: self.ongoing_activations = 0 self.running_flag.value = 1 self._start_invoker_process() log_msg = ('ExecutorID {} | JobID {} - Starting function ' 'invocation: {}() - Total: {} activations' .format(job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) if not self.log_active: print(log_msg) if self.ongoing_activations < self.workers: callids = range(job.total_calls) total_direct = self.workers-self.ongoing_activations callids_to_invoke_direct = callids[:total_direct] callids_to_invoke_nondirect = callids[total_direct:] self.ongoing_activations += len(callids_to_invoke_direct) logger.debug('ExecutorID {} | JobID {} - Free workers: ' '{} - Going to invoke {} function activations' .format(job.executor_id, job.job_id, total_direct, len(callids_to_invoke_direct))) def _callback(future): future.result() executor = ThreadPoolExecutor(job.invoke_pool_threads) for i in callids_to_invoke_direct: call_id = "{:05d}".format(i) future = executor.submit(self._invoke, job, call_id) future.add_done_callback(_callback) time.sleep(0.1) # Put into the queue the rest of the callids to invoke within the process if callids_to_invoke_nondirect: logger.debug('ExecutorID {} | JobID {} - Putting remaining ' '{} function invocations into pending queue' .format(job.executor_id, job.job_id, len(callids_to_invoke_nondirect))) for i in callids_to_invoke_nondirect: call_id = "{:05d}".format(i) self.pending_calls_q.put((job, call_id)) else: logger.debug('ExecutorID {} | JobID {} - Ongoing activations ' 'reached {} workers, queuing {} function invocations' .format(job.executor_id, job.job_id, self.workers, job.total_calls)) for i in range(job.total_calls): call_id = "{:05d}".format(i) self.pending_calls_q.put((job, call_id)) self.job_monitor.start_job_monitoring(job) except (KeyboardInterrupt, Exception) as e: self.stop() raise e # Create all futures futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures
def run(self, job): """ Run a job described in job_description """ job.runtime_name = self.runtime_name try: while True: self.token_bucket_q.get_nowait() self.ongoing_activations -= 1 except Exception: pass self.prometheus.send_metric(name='job_total_calls', value=job.total_calls, labels=(('executor_id', job.executor_id), ('job_id', job.job_id), ('function_name', job.function_name))) if self.remote_invoker: """ Remote Invocation Use a single cloud function to perform all the function invocations """ log_level = logger.getEffectiveLevel() logging.getLogger('lithops').setLevel(logging.CRITICAL) self.select_runtime(job.job_id, self.REMOTE_INVOKER_MEMORY) logging.getLogger('lithops').setLevel(log_level) log_msg = ('ExecutorID {} | JobID {} - Starting function ' 'invocation: {}() - Total: {} activations'.format( job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) self._invoke_remote(job) # th = Thread(target=self._invoke_remote, args=(job,), daemon=True) # th.start() else: """ Normal Invocation Use local threads to perform all the function invocations """ try: if self.running_flag.value == 0: self.running_workers = 0 self.running_flag.value = 1 self._start_invoker_process() log_msg = ('ExecutorID {} | JobID {} - Starting function ' 'invocation: {}() - Total: {} activations'.format( job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) logger.debug('ExecutorID {} | JobID {} - Chunksize:' ' {} - Worker processes: {}'.format( job.executor_id, job.job_id, job.chunksize, job.worker_processes)) if self.running_workers < self.workers: free_workers = self.workers - self.running_workers total_direct = free_workers * job.chunksize callids = range(job.total_calls) callids_to_invoke_direct = callids[:total_direct] callids_to_invoke_nondirect = callids[total_direct:] ci = len(callids_to_invoke_direct) cz = job.chunksize consumed_workers = ci // cz + (ci % cz > 0) self.running_workers += consumed_workers logger.debug( 'ExecutorID {} | JobID {} - Free workers:' ' {} - Going to run {} activations in {} workers'. format(job.executor_id, job.job_id, free_workers, len(callids_to_invoke_direct), consumed_workers)) def _callback(future): future.result() executor = ThreadPoolExecutor(job.invoke_pool_threads) for call_ids_range in iterchunks(callids_to_invoke_direct, job.chunksize): future = executor.submit(self._invoke, job, call_ids_range) future.add_done_callback(_callback) time.sleep(0.1) # Put into the queue the rest of the callids to invoke within the process if callids_to_invoke_nondirect: logger.debug( 'ExecutorID {} | JobID {} - Putting remaining ' '{} function activations into pending queue'. format(job.executor_id, job.job_id, len(callids_to_invoke_nondirect))) for call_ids_range in iterchunks( callids_to_invoke_nondirect, job.chunksize): self.pending_calls_q.put((job, call_ids_range)) else: logger.debug( 'ExecutorID {} | JobID {} - Reached maximun {} ' 'workers, queuing {} function activations'.format( job.executor_id, job.job_id, self.workers, job.total_calls)) for call_ids_range in iterchunks(job.total_calls, job.chunksize): self.pending_calls_q.put((job, call_ids_range)) self.job_monitor.start_job_monitoring(job) except (KeyboardInterrupt, Exception) as e: self.stop() raise e log_file = os.path.join(LOGS_DIR, job.job_key + '.log') logger.info( "ExecutorID {} | JobID {} - View execution logs at {}".format( job.executor_id, job.job_id, log_file)) # Create all futures futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures
def run(self, job_description): """ Run a job described in job_description """ job_description['runtime_name'] = self.config['serverless']['runtime'] job_description['runtime_memory'] = self.config['serverless'][ 'runtime_memory'] job_description['runtime_timeout'] = self.config['serverless'][ 'runtime_timeout'] execution_timeout = job_description['execution_timeout'] runtime_timeout = self.config['serverless']['runtime_timeout'] if execution_timeout >= runtime_timeout: job_description['execution_timeout'] = runtime_timeout - 5 job = SimpleNamespace(**job_description) try: while True: self.token_bucket_q.get_nowait() self.ongoing_activations -= 1 except Exception: pass if self.remote_invoker: old_stdout = sys.stdout sys.stdout = open(os.devnull, 'w') self.select_runtime(job.job_id, REMOTE_INVOKER_MEMORY) sys.stdout = old_stdout log_msg = ( 'ExecutorID {} | JobID {} - Starting remote function invocation: {}() ' '- Total: {} activations'.format(job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) if not self.log_active: print(log_msg) th = Thread(target=self._invoke_remote, args=(job_description, )) th.daemon = True th.start() time.sleep(0.1) else: try: if self.running_flag.value == 0: self.ongoing_activations = 0 self.running_flag.value = 1 self._start_invoker_process() log_msg = ( 'ExecutorID {} | JobID {} - Starting function invocation: {}() - Total: {} ' 'activations'.format(job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) if not self.log_active: print(log_msg) if self.ongoing_activations < self.workers: callids = range(job.total_calls) total_direct = self.workers - self.ongoing_activations callids_to_invoke_direct = callids[:total_direct] callids_to_invoke_nondirect = callids[total_direct:] self.ongoing_activations += len(callids_to_invoke_direct) logger.debug( 'ExecutorID {} | JobID {} - Free workers: {} - Going to invoke {} function activations' .format(job.executor_id, job.job_id, total_direct, len(callids_to_invoke_direct))) with ThreadPoolExecutor( max_workers=job.invoke_pool_threads) as executor: for i in callids_to_invoke_direct: call_id = "{:05d}".format(i) executor.submit(self._invoke, job, call_id) # Put into the queue the rest of the callids to invoke within the process if callids_to_invoke_nondirect: logger.debug( 'ExecutorID {} | JobID {} - Putting remaining {} function invocations into pending queue' .format(job.executor_id, job.job_id, len(callids_to_invoke_nondirect))) for i in callids_to_invoke_nondirect: call_id = "{:05d}".format(i) self.pending_calls_q.put((job, call_id)) else: logger.debug( 'ExecutorID {} | JobID {} - Ongoing activations reached {} workers, ' 'putting {} function invocations into pending queue'. format(job.executor_id, job.job_id, self.workers, job.total_calls)) for i in range(job.total_calls): call_id = "{:05d}".format(i) self.pending_calls_q.put((job, call_id)) self.job_monitor.start_job_monitoring(job) except (KeyboardInterrupt, Exception) as e: self.stop() raise e # Create all futures futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job_description, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures