def get_gpus(docker_manager, gpu_settings, gpu_ids): """ Returns a list of gpus which are sufficient for the given gpu settings. Otherwise raise an Exception :param docker_manager: The DockerManager used to query gpus :type docker_manager: DockerManager :param gpu_settings: The gpu settings of the red experiment specifying the required gpus :type gpu_settings: Dict :param gpu_ids: The gpu_ids specified by the user to use for the execution. If None all gpus are considered. :type gpu_ids: List[int] or None :return: A list of GPUDevices to use for this experiment :rtype: List[GPUDevice] :raise InsufficientGPUError: If GPU settings could not be fulfilled """ gpus = None gpu_requirements = get_gpu_requirements(gpu_settings) # dont do anything, if no gpus are required if gpu_requirements or gpu_ids: gpu_devices = get_gpu_devices(docker_manager, gpu_ids) gpus = match_gpus(gpu_devices, gpu_requirements) # if gpu_ids are specified, ignore gpu matching if gpu_ids: gpus = gpu_devices return gpus
def _node_possibly_sufficient(node, experiment): """ Returns True if the node could be sufficient for the experiment, even if the node does not have sufficient hardware at the moment (because of running batches). :param node: The node to check :type node: CompleteNode :param experiment: The experiment for which the node is sufficient or not. :return: True, if the node is possibly sufficient otherwise False """ # check if node is initialized if (node.ram is None) or (node.gpus is None): return False if node.ram < experiment['container']['settings']['ram']: return False gpu_requirements = get_gpu_requirements( experiment['container']['settings'].get('gpus')) try: match_gpus(node.gpus, gpu_requirements) except InsufficientGPUError: return False return True
def _node_sufficient(node, experiment): """ Returns True if the nodes hardware is sufficient for the experiment :param node: The node to test :type node: CompleteNode :param experiment: A dictionary containing hardware requirements for the experiment :return: True, if the nodes hardware is sufficient for the experiment, otherwise False """ if not node.online: return False if node.ram_available < experiment['container']['settings']['ram']: return False # check gpus gpu_requirements = get_gpu_requirements( experiment['container']['settings'].get('gpus')) try: _gpus = match_gpus(node.gpus_available, gpu_requirements) except InsufficientGPUError: return False return True
def _schedule_batch(self, next_batch, nodes, batch_count_cache): """ Tries to find a node that is capable of processing the given batch. If no capable node could be found, None is returned. If a node was found, that is capable of processing the given batch, this node is written to the node property of the batch. The batches state is then updated to 'scheduled'. :param next_batch: The batch to schedule. :param nodes: The nodes on which the batch should be scheduled. :type nodes: List[CompleteNode] :param batch_count_cache: A dictionary mapping experiment ids to the number of batches of this experiment, which in state processing or scheduled. This dictionary is allowed to overestimate the number of batches. :type batch_count_cache: Dict[str, int] :return: The name of the node on which the given batch is scheduled If the batch could not be scheduled None is returned :raise TrusteeServiceError: If the trustee service is unavailable. """ batch_id = str(next_batch['_id']) experiment_id = next_batch['experimentId'] try: experiment = self._get_experiment_of_batch(experiment_id) except Exception as e: batch_failure(self._mongo, batch_id, repr(e), None, next_batch['state'], disable_retry_if_failed=True) return None ram = experiment['container']['settings']['ram'] # limit the number of currently executed batches from a single experiment concurrency_limit = experiment.get('execution', {}).get( 'settings', {}).get('batchConcurrencyLimit', 64) # number of batches which are scheduled or processing of the given experiment batch_count = self._get_number_of_batches_of_experiment( experiment_id, batch_count_cache) if batch_count >= concurrency_limit: return None # check impossible experiments if not Scheduler._check_nodes_possibly_sufficient(nodes, experiment): debug_info = 'There are no nodes configured that are possibly sufficient for experiment "{}"' \ .format(next_batch['experimentId']) batch_failure(self._mongo, batch_id, debug_info, None, next_batch['state'], disable_retry_if_failed=True) return None # select node selected_node = Scheduler._get_best_node(nodes, experiment) if selected_node is None: return None # calculate ram / gpus selected_node.ram_available -= ram used_gpu_ids = None if selected_node.gpus_available: gpu_requirements = get_gpu_requirements( experiment['container']['settings'].get('gpus')) available_gpus = selected_node.gpus_available used_gpus = match_gpus(available_gpus, requirements=gpu_requirements) used_gpu_ids = [] for gpu in used_gpus: used_gpu_ids.append(gpu.device_id) available_gpus.remove(gpu) # check mounting mount_connectors = red_get_mount_connectors_from_inputs( next_batch['inputs']) is_mounting = bool(mount_connectors) allow_insecure_capabilities = self._conf.d['controller']['docker'].get( 'allow_insecure_capabilities', False) if not allow_insecure_capabilities and is_mounting: # set state to failed, because insecure_capabilities are not allowed but needed, by this batch. debug_info = 'FUSE support for this agency is disabled, but the following input/output-keys are ' \ 'configured to mount inside a docker container.{}{}'.format(os.linesep, mount_connectors) batch_failure(self._mongo, batch_id, debug_info, None, next_batch['state'], disable_retry_if_failed=True) return None # update batch data update_result = self._mongo.db['batches'].update_one( { '_id': next_batch['_id'], 'state': next_batch['state'] }, { '$set': { 'state': 'scheduled', 'node': selected_node.node_name, 'usedGPUs': used_gpu_ids, 'mount': is_mounting }, '$push': { 'history': { 'state': 'scheduled', 'time': time(), 'debugInfo': None, 'node': selected_node.node_name, 'ccagent': None, 'dockerStats': None } }, '$inc': { 'attempts': 1 } }) if update_result.modified_count == 1: # The state of the scheduled batch switched from 'registered' to 'scheduled', so increase the batch_count. # batch_count_cache always contains experiment_id, because _get_number_of_batches_of_experiment() # always inserts the given experiment_id batch_count_cache[experiment_id] += 1 return selected_node.node_name else: return None