コード例 #1
0
class ExperimentClient:
    def __init__(self, host):
        """
        Submarine experiment client constructor
        :param host: An HTTP URI like http://submarine-server:8080.
        """
        # TODO(pingsutw): support authentication for talking to the submarine server
        self.host = host
        configuration = Configuration()
        configuration.host = host + '/api'
        api_client = ApiClient(configuration=configuration)
        self.experiment_api = ExperimentApi(api_client=api_client)

    def create_experiment(self, experiment_spec):
        """
        Create an experiment
        :param experiment_spec: submarine experiment spec
        :return: submarine experiment
        """
        response = self.experiment_api.create_experiment(
            experiment_spec=experiment_spec)
        return response.result

    def wait_for_finish(self, id, polling_interval=10):
        """
        Waits until experiment is finished or failed
        :param id: submarine experiment id
        :param polling_interval: How often to poll for the status of the experiment.
        :return: str: experiment logs
        """
        index = 0
        while True:
            status = self.get_experiment(id)['status']
            if status == 'Succeeded' or status == 'Deleted':
                self._log_pod(id, index)
                break
            index = self._log_pod(id, index)
            time.sleep(polling_interval)

    def _log_pod(self, id, index):
        response = self.experiment_api.get_log(id)
        log_content = response.result['logContent'][0]
        for i, log in enumerate(log_content['podLog']):
            if i < index:
                continue
            index += 1
            logging.info("%s", log)
        return index

    def patch_experiment(self, id, experiment_spec):
        """
        Patch an experiment
        :param id: submarine experiment id
        :param experiment_spec: submarine experiment spec
        :return: submarine experiment
        """
        response = self.experiment_api.patch_experiment(
            id=id, experiment_spec=experiment_spec)
        return response.result

    def get_experiment(self, id):
        """
        Get the experiment's detailed info by id
        :param id: submarine experiment id
        :return: submarine experiment
        """
        response = self.experiment_api.get_experiment(id=id)
        return response.result

    def list_experiments(self, status):
        """
        List all experiment for the user
        :param status: Accepted, Created, Running, Succeeded, Deleted
        :return: List of submarine experiments
        """
        response = self.experiment_api.list_experiments(status=status)
        return response.result

    def delete_experiment(self, id):
        """
        Delete the Submarine experiment
        :param id: Submarine experiment id
        :return: The detailed info about deleted submarine experiment
        """
        response = self.experiment_api.delete_experiment(id)
        return response.result

    def get_log(self, id, master=True):
        """
        Get training logs of the experiment.
        By default only get the logs of Pod that has labels 'job-role: master'.
        :param master: By default get pod with label 'job-role: master' pod if True.
                    If need to get more Pod Logs, set False.
        :param id: Experiment ID
        :return: str: experiment logs
        """
        response = self.experiment_api.get_log(id)
        log_contents = response.result['logContent']

        if master is True:
            log_contents = [log_contents[0]]

        for log_content in log_contents:
            logging.info("The logs of Pod %s:\n", log_content['podName'])
            for log in log_content['podLog']:
                logging.info("%s", log)

    def list_log(self, status):
        """
        List experiment log
        :param status: Accepted, Created, Running, Succeeded, Deleted
        :return: List of submarine log
        """
        response = self.experiment_api.list_log(status=status)
        return response.result
コード例 #2
0
class ExperimentClient:
    def __init__(self, host=generate_host()):
        """
        Submarine experiment client constructor
        :param host: An HTTP URI like http://submarine-server:8080.
        """
        # TODO(pingsutw): support authentication for talking to the submarine server
        self.host = host
        configuration = Configuration()
        configuration.host = host + '/api'
        api_client = ApiClient(configuration=configuration)
        self.experiment_api = ExperimentApi(api_client=api_client)

    def create_experiment(self, experiment_spec):
        """
        Create an experiment
        :param experiment_spec: submarine experiment spec
        :return: submarine experiment
        """
        response = self.experiment_api.create_experiment(experiment_spec=experiment_spec)
        return response.result

    def wait_for_finish(self, id, polling_interval=10):
        """
        Waits until experiment is finished or failed
        :param id: submarine experiment id
        :param polling_interval: How often to poll for the status of the experiment.
        :return: str: experiment logs
        """
        index = 0
        while True:
            status = self.get_experiment(id)['status']
            if status == 'Succeeded' or status == 'Deleted':
                self._log_pod(id, index)
                break
            index = self._log_pod(id, index)
            time.sleep(polling_interval)

    def _log_pod(self, id, index):
        response = self.experiment_api.get_log(id)
        log_contents = response.result['logContent']
        if len(log_contents) == 0:
            return index
        log_content = log_contents[0]
        for i, log in enumerate(log_content['podLog']):
            if i < index:
                continue
            index += 1
            logging.info("%s", log)
        return index

    def patch_experiment(self, id, experiment_spec):
        """
        Patch an experiment
        :param id: submarine experiment id
        :param experiment_spec: submarine experiment spec
        :return: submarine experiment
        """
        response = self.experiment_api.patch_experiment(id=id, experiment_spec=experiment_spec)
        return response.result

    def get_experiment(self, id):
        """
        Get the experiment's detailed info by id
        :param id: submarine experiment id
        :return: submarine experiment
        """
        response = self.experiment_api.get_experiment(id=id)
        return response.result

    def list_experiments(self, status=None):
        """
        List all experiment for the user
        :param status: Accepted, Created, Running, Succeeded, Deleted
        :return: List of submarine experiments
        """
        response = self.experiment_api.list_experiments(status=status)
        return response.result

    def delete_experiment(self, id):
        """
        Delete the Submarine experiment
        :param id: Submarine experiment id
        :return: The detailed info about deleted submarine experiment
        """
        response = self.experiment_api.delete_experiment(id)
        return response.result

    def get_log(self, id, onlyMaster=False):
        """
        Get training logs of all pod of the experiment.
        By default get all the logs of Pod
        :param id: experiment id
        :param onlyMaster: By default include pod log of "master" which might be
         Tensorflow PS/Chief or PyTorch master
        :return: str: pods logs
        """
        response = self.experiment_api.get_log(id)
        log_contents = response.result['logContent']

        if onlyMaster is True and len(log_contents) != 0:
            log_contents = [log_contents[0]]

        for log_content in log_contents:
            logging.info("The logs of Pod %s:\n", log_content['podName'])
            for log in log_content['podLog']:
                logging.info("%s", log)

    def list_log(self, status):
        """
        List experiment log
        :param status: Accepted, Created, Running, Succeeded, Deleted
        :return: List of submarine log
        """
        response = self.experiment_api.list_log(status=status)
        return response.result