def __init__(self, denominator, max_width=None, eta_every=1):
     self._eta = ETA(denominator=denominator)
     self.max_width = max_width
     self.eta_every = eta_every
     self.force_done = False
     self._eta_string = ''
     self._eta_count = 1
예제 #2
0
def test_linear_slope_1():
    eta = ETA(100)
    eta._timing_data = deque([(10, 10), (20, 20), (30, 30), (40, 40)])
    getattr(eta, '_calculate')()

    assert 100 == eta.eta_epoch
    assert 1.0 == eta.rate
    assert 1.0 == eta.rate_unstable
예제 #3
0
def test_linear_transform_undefined():
    eta = ETA()
    eta._timing_data = deque([(1.2, 22), (2.4, 58), (3.1, 102), (4.4, 118)])
    getattr(eta, '_calculate')()

    assert eta.eta_epoch is None
    assert 30 < eta.rate < 35
    assert 12 < eta.rate_unstable < 13
예제 #4
0
def test_linear_transform():
    """Wolfram Alpha:
    x is the timestamp. y is the numerator. 120 is the denominator.
    linear fit {1.2, 22},{2.4, 58},{3.1, 102},{4.4, 118}

    The closer we get to 100%, the more vertical shift/transform is applied to the line.
    As we near the end we want the line to get closer to the last point on the graph.
    This avoids having 99% with an ETA in the past.
    """
    eta = ETA(120)
    eta._timing_data = deque([(1.2, 22), (2.4, 58), (3.1, 102), (4.4, 118)])
    getattr(eta, '_calculate')()

    assert 4.4 < eta.eta_epoch < 4.6
    assert 30 < eta.rate < 35
    assert 12 < eta.rate_unstable < 13
예제 #5
0
 def __init__(self, denominator, max_width=None, eta_every=1):
     self._eta = ETA(denominator=denominator)
     self.max_width = max_width
     self.eta_every = eta_every
     self.force_done = False
     self._eta_string = ''
     self._eta_count = 1
예제 #6
0
    def gather_results(self, workers, log_queue, test_split=False):
        '''
        check for logs while waiting for workers
        '''
        results = {}
        eval_type = 'subgoal' if self.args.subgoals else 'task'
        lock = filelock.FileLock(self.results_path + '.lock')
        eta = ETA(self.num_trials, scope=32)
        while True:
            if log_queue.qsize() > 0:
                # there is a new log entry available, process it
                log_entry, trial_uid, model_path = log_queue.get()
                # load old results (if available)
                with lock:
                    if os.path.exists(self.results_path):
                        with open(self.results_path, 'r') as results_file:
                            results = json.load(results_file)

                eval_epoch = os.path.basename(model_path)
                # update the old results with the new log entry
                if eval_epoch not in results:
                    results[eval_epoch] = {}
                if eval_type not in results[eval_epoch]:
                    results[eval_epoch][eval_type] = {}
                if trial_uid in results[eval_epoch][eval_type] and not test_split:
                    success_prev = results[eval_epoch][eval_type][trial_uid]['success']
                    success_curr = log_entry['success']
                    if success_prev != success_curr:
                        print(colored(
                            'WARNING: trial {} result has changed from {} to {}'.format(
                                trial_uid,
                                'success' if success_prev else 'fail',
                                'success' if success_curr else 'fail'), 'yellow'))
                results[eval_epoch][eval_type][trial_uid] = log_entry

                # print updated results
                self.num_trials_done += 1
                eta.numerator = self.num_trials_done
                if not test_split:
                    successes = [
                        log['success'] for log in results[eval_epoch][eval_type].values()]
                    print(colored(
                        '{:4d}/{} trials are done (current SR = {:.1f}), ETA = {}, elapsed = {}'.format(
                            self.num_trials_done,
                            self.num_trials,
                            100 * sum(successes) / len(successes),
                            time.strftime('%H:%M:%S', time.gmtime(eta.eta_seconds)),
                            time.strftime('%H:%M:%S', time.gmtime(eta.elapsed))),
                        'green'))
                # make a backup copy of results file before writing
                eval_util.save_with_backup(results, self.results_path, lock)
                # update info.json file
                model_util.update_log(
                    self.args.dout, stage='eval',
                    update='increase', progress=1)

            # check whether all workers have exited (exitcode == None means they are still running)
            all_terminated = all([worker.exitcode is not None for worker in workers])
            if all_terminated and log_queue.qsize() == 0:
                if self.num_trials_left > 0:
                    print(colored('WARNING: only {}/{} trials were evaluated'.format(
                        self.num_trials_done, self.num_trials), 'red'))
                # our mission is over
                break
            time.sleep(1)
        print(colored('Evaluation is complete', 'green'))
예제 #7
0
class BaseProgressBar(object):
    """Holds common properties/methods/etc for ProgressBar and related subclasses."""

    def __init__(self, denominator, max_width=None, eta_every=1):
        self._eta = ETA(denominator=denominator)
        self.max_width = max_width
        self.eta_every = eta_every
        self.force_done = False
        self._eta_string = ''
        self._eta_count = 1

    @staticmethod
    def _generate_eta(seconds):
        """Kind of like an interface method, to be implemented by subclasses."""
        raise NotImplementedError

    @property
    def denominator(self):
        """Returns the denominator as an integer."""
        return int(self._eta.denominator)

    @property
    def done(self):
        """Returns True if the progress has completed."""
        if self.force_done:
            return True
        return self._eta.done

    @property
    def numerator(self):
        """Returns the numerator as an integer."""
        return int(self._eta.numerator)

    @numerator.setter
    def numerator(self, value):
        """Sets a new numerator and generates the ETA. Must be greater than or equal to previous numerator."""
        # If ETA is every iteration, don't do anything fancy.
        if self.eta_every <= 1:
            self._eta.numerator = value
            self._eta_string = self._generate_eta(self._eta.eta_seconds)
            return

        # If ETA is not every iteration, unstable rate is used. If this bar is undefined, no point in calculating ever.
        if self._eta.undefined:
            self._eta.set_numerator(value, calculate=False)
            return

        # Calculate if this iteration is the right one.
        if self._eta_count >= self.eta_every:
            self._eta_count = 1
            self._eta.numerator = value
            self._eta_string = self._generate_eta(self._eta.eta_seconds)
            return

        self._eta_count += 1
        self._eta.set_numerator(value, calculate=False)

    @property
    def percent(self):
        """Returns the percent as a float."""
        return float(self._eta.percent)

    @property
    def rate(self):
        """Returns the rate of the progress as a float. Selects the unstable rate if eta_every > 1 for performance."""
        return float(self._eta.rate_unstable if self.eta_every > 1 else self._eta.rate)

    @property
    def undefined(self):
        """Return True if the progress bar is undefined (unknown denominator)."""
        return self._eta.undefined
class BaseProgressBar(object):
    """Holds common properties/methods/etc for ProgressBar and related subclasses."""
    def __init__(self, denominator, max_width=None, eta_every=1):
        self._eta = ETA(denominator=denominator)
        self.max_width = max_width
        self.eta_every = eta_every
        self.force_done = False
        self._eta_string = ''
        self._eta_count = 1

    @staticmethod
    def _generate_eta(seconds):
        """Kind of like an interface method, to be implemented by subclasses."""
        raise NotImplementedError

    @property
    def denominator(self):
        """Returns the denominator as an integer."""
        return int(self._eta.denominator)

    @property
    def done(self):
        """Returns True if the progress has completed."""
        if self.force_done:
            return True
        return self._eta.done

    @property
    def numerator(self):
        """Returns the numerator as an integer."""
        return int(self._eta.numerator)

    @numerator.setter
    def numerator(self, value):
        """Sets a new numerator and generates the ETA. Must be greater than or equal to previous numerator."""
        # If ETA is every iteration, don't do anything fancy.
        if self.eta_every <= 1:
            self._eta.numerator = value
            self._eta_string = self._generate_eta(self._eta.eta_seconds)
            return

        # If ETA is not every iteration, unstable rate is used. If this bar is undefined, no point in calculating ever.
        if self._eta.undefined:
            self._eta.set_numerator(value, calculate=False)
            return

        # Calculate if this iteration is the right one.
        if self._eta_count >= self.eta_every:
            self._eta_count = 1
            self._eta.numerator = value
            self._eta_string = self._generate_eta(self._eta.eta_seconds)
            return

        self._eta_count += 1
        self._eta.set_numerator(value, calculate=False)

    @property
    def percent(self):
        """Returns the percent as a float."""
        return float(self._eta.percent)

    @property
    def rate(self):
        """Returns the rate of the progress as a float. Selects the unstable rate if eta_every > 1 for performance."""
        return float(
            self._eta.rate_unstable if self.eta_every > 1 else self._eta.rate)

    @property
    def undefined(self):
        """Return True if the progress bar is undefined (unknown denominator)."""
        return self._eta.undefined