Пример #1
0
    def _initialize(self):
        """Tries to initialize the task."""

        self.log("Running initialize")
        self.initialize_called = True
        while True:
            public_ip = self.public_ip  # todo: add retry logic to public_ip property
            if public_ip:
                break
            print(
                f"Trying to initialize, but task {self.name} doesn't have public_ip, sleeping"
            )
            time.sleep(TIMEOUT_SEC)

        while True:
            self.ssh_client = u.ssh_to_host(self.public_ip, self.keypair_fn,
                                            self.username)
            if self.ssh_client is None:
                self.log("SSH into %s:%s failed, retrying in %d seconds" %
                         (self.job.name, self.id, TIMEOUT_SEC))
                time.sleep(TIMEOUT_SEC)
            else:
                break

        # todo: install tmux
        self._setup_tmux()
        self.run('mkdir -p ' + self.remote_scratch)
        if not self.skip_efs_mount:
            self._mount_efs()

        # run initialization commands here
        if self._is_initialized_file_present():
            self.log("reusing previous initialized state")
        elif self.install_script:
            self.log("running install script")

            self.install_script += '\necho ok > /tmp/is_initialized\n'
            self.file_write('install.sh', u._add_echo(self.install_script))
            self.run('bash -e install.sh', max_wait_sec=2400)  # fail on errors
            # TODO(y): propagate error messages printed on console to the user
            # right now had to log into tmux to see it
            assert self._is_initialized_file_present()
        else:
            self.log('No install script. Skipping to end')
            # installation happens through user-data instead of install script
            # if neither one is passed, manually create is_initialized
            self.run('echo ok > /tmp/is_initialized')

        self.connect_instructions = """
ssh -i %s -o StrictHostKeyChecking=no %s@%s
tmux a
""".strip() % (self.keypair_fn, self.username, self.public_ip)
        self.initialized = True
        self.log("Initialize complete")
        self.log(self.connect_instructions)
Пример #2
0
    def _initialize(self):
        """Tries to initialize the task."""

        self.log("Running initialize")
        self.initialize_called = True
        public_ip = self.public_ip  # todo: add retry logic to public_ip property

        while True:
            self.ssh_client = u.ssh_to_host(self.public_ip, self.keypair_fn,
                                            self.username)
            if self.ssh_client is None:
                self.log("SSH into %s:%s failed, retrying in %d seconds" %
                         (self.job.name, self.id, TIMEOUT_SEC))
                time.sleep(TIMEOUT_SEC)
            else:
                break

        # todo: install tmux
        self._setup_tmux()
        self.run('mkdir -p ' + self.remote_scratch)
        self._mount_efs()

        # run initialization commands here
        if self._is_initialized_file_present():
            self.log("reusing previous initialized state")
        else:
            self.log("running install script")

            self.install_script += '\necho ok > /tmp/is_initialized\n'
            self.file_write('install.sh', u._add_echo(self.install_script))
            self.run('bash -e install.sh')  # fail on errors
            # TODO(y): propagate error messages printed on console to the user
            # right now had to log into tmux to see it

        assert self._is_initialized_file_present()

        self.connect_instructions = """
ssh -i %s -o StrictHostKeyChecking=no %s@%s
tmux a
""".strip() % (self.keypair_fn, self.username, self.public_ip)
        self.log("Initialize complete")