Exemple #1
0
def decide(detmaster: str, experiment_id: int, model_name: str) -> bool:
    # Submit determined experiment via CLI
    from determined.experimental import Determined
    import os

    os.environ['DET_MASTER'] = detmaster

    def get_validation_metric(checkpoint):
        metrics = checkpoint.validation['metrics']
        config = checkpoint.experiment_config
        searcher = config['searcher']
        smaller_is_better = bool(searcher['smaller_is_better'])
        metric_name = searcher['metric']
        if 'validation_metrics' in metrics:
            metric = metrics['validation_metrics'][metric_name]
        else:
            metric = metrics['validationMetrics'][metric_name]
        return (metric, smaller_is_better)

    d = Determined()
    checkpoint = d.get_experiment(experiment_id).top_checkpoint()
    metric, smaller_is_better = get_validation_metric(checkpoint)

    models = d.get_models(name=model_name)
    model = None
    for m in models:
        if m.name == model_name:
            model = m
            break
    if not model:
        print(f'Registering new Model: {model_name}')
        model = Determined().create_model(model_name)
        model.register_version(checkpoint.uuid)
        better = True
    else:
        latest_version = model.get_version()
        if latest_version is None:
            print(f'Registering new version: {model_name}')
            model.register_version(checkpoint.uuid)
            better = True
        else:
            old_metric, _ = get_validation_metric(latest_version)
            if smaller_is_better:
                if metric < old_metric:
                    print(f'Registering new version: {model_name}')
                    model.register_version(checkpoint.uuid)
                    better = True
                else:
                    better = False
            else:
                if metric > old_metric:
                    print(f'Registering new version: {model_name}')
                    model.register_version(checkpoint.uuid)
                    better = True
                else:
                    better = False

    if not better:
        print('Previous model version was better, logging...')
    return better
def main():
    parser = argparse.ArgumentParser(description='Run Determined Example')
    parser.add_argument('experiment_id',
                        type=str,
                        help='path to context directory')
    parser.add_argument('model_name',
                        type=str,
                        help='path to context directory')
    args = parser.parse_args()

    checkpoint = Determined().get_experiment(
        args.experiment_id).top_checkpoint()
    metric, smaller_is_better = get_validation_metric(checkpoint)

    models = Determined().get_models(name=args.model_name)
    model = None
    for m in models:
        if m.name == args.model_name:
            model = m
            break
    if not model:
        print(f'Registering new Model: {args.model_name}')
        model = Determined().create_model(args.model_name)
        model.register_version(checkpoint.uuid)
        better = True
    else:
        latest_version = model.get_version()
        if latest_version is None:
            print(f'Registering new version: {args.model_name}')
            model.register_version(checkpoint.uuid)
            better = True
        else:
            old_metric, _ = get_validation_metric(latest_version)
            if smaller_is_better:
                if metric < old_metric:
                    print(f'Registering new version: {args.model_name}')
                    model.register_version(checkpoint.uuid)
                    better = True
                else:
                    better = False
            else:
                if metric > old_metric:
                    print(f'Registering new version: {args.model_name}')
                    model.register_version(checkpoint.uuid)
                    better = True
                else:
                    better = False

    if not better:
        print('Previous model version was better, logging...')
    # Write experiment id to output file
    with open('/tmp/decision.txt', 'w') as f:
        if better:
            f.write('yes')
        else:
            f.write('no')