Esempio n. 1
0
def install_xcode():
    result = subprocess.call(('xcode-select', '--install'),
                             stderr=subprocess.PIPE)
    if result > 0:
        print(ansi.success('CHECK') + ' xcode-select --install')
    else:
        print(ansi.success('INSTALL') + ' xcode command line tools')
Esempio n. 2
0
def install_pyenv():
    virtualenv = os.path.join(env.PYENV, 'plugins', 'pyenv-virtualenv')
    if os.path.exists(env.BIN_PYENV) and os.path.exists(virtualenv):
        return

    if os.path.exists(env.PYENV) and not os.path.isfile(env.BIN_PYENV):
        print(ansi.warning() +
              ' pyenv executable is not present at %s' % env.BIN_PYENV)
        while True:
            answer = input(
                'Would you like to blow away ~/.pyenv and rebuild from scratch? [Y/n] '
            )
            if answer in ['', 'y', 'Y']:
                shutil.rmtree(lore.env.PYENV)
                break
            elif answer in ['n', 'N']:
                sys.exit(ansi.error() + ' please fix pyenv before continuing')
            else:
                print('please enter Y or N')

    if not os.path.exists(env.PYENV):
        print(ansi.success('INSTALLING') + ' pyenv')
        subprocess.check_call(
            ('git', 'clone', 'https://github.com/pyenv/pyenv.git', env.PYENV))
    else:
        print(ansi.success('CHECK') + ' existing pyenv installation')
    env.set_python_version(env.PYTHON_VERSION)

    if not os.path.exists(virtualenv):
        print(ansi.success('INSTALLING') + ' pyenv virtualenv')
        subprocess.check_call(
            ('git', 'clone', 'https://github.com/pyenv/pyenv-virtualenv.git',
             virtualenv))
    else:
        print(ansi.success('CHECK') + ' existing virtualenv installation')
Esempio n. 3
0
def install_tensorflow():
    description = subprocess.check_output(
        (env.bin_python, '-m', 'pip', 'show', 'tensorflow')).decode('utf-8')
    version = re.match('.*^Version: ([^\n]+)', description,
                       re.S | re.M).group(1)
    if not version:
        sys.exit(ansi.error() + ' tensorflow is not in requirements.txt')

    print(ansi.success('NATIVE') + ' tensorflow ' + version)

    python_version = ''.join(env.python_version.split('.')[0:2])
    cached = os.path.join(
        env.pyenv, 'cache', 'tensorflow_pkg',
        'tensorflow-' + version + '-cp' + python_version + '*')

    paths = glob.glob(cached)

    if not paths:
        build_tensorflow(version)
        paths = glob.glob(cached)

    path = paths[0]

    subprocess.check_call(
        (env.bin_python, '-m', 'pip', 'uninstall', '-y', 'tensorflow'))
    print(ansi.success('INSTALL') + ' tensorflow native build')
    subprocess.check_call((env.bin_python, '-m', 'pip', 'install', path))
Esempio n. 4
0
def fit(parsed, unknown):
    print(ansi.success('FITTING ') + parsed.model)
    Model = _get_fully_qualified_class(parsed.model)
    model = Model()

    valid_model_fit_args = _get_valid_fit_args(model.fit)
    valid_estimator_fit_args = _get_valid_fit_args(model.estimator.fit)
    valid_fit_args = valid_model_fit_args.args[1:] + valid_estimator_fit_args.args[1:]

    model_attrs = _filter_private_attributes(model.__dict__)
    pipeline_attrs = _filter_private_attributes(model.pipeline.__dict__)
    estimator_attrs = _filter_private_attributes(model.estimator.__dict__)
    estimator_attrs.pop('model', None)

    grouped, unpaired = _pair_args(unknown)

    # assign args to their receivers
    fit_args = {}
    unknown_args = []
    for name, value in grouped:
        if name in model_attrs:
            value = _cast_attr(value, getattr(model, name))
            setattr(model, name, value)
        elif name in pipeline_attrs:
            value = _cast_attr(value, getattr(model.pipeline, name))
            setattr(model.pipeline, name, value)
        elif name in estimator_attrs:
            value = _cast_attr(value, getattr(model.estimator, name))
            setattr(model.estimator, name, value)
        elif name in valid_model_fit_args.args:
            index = valid_model_fit_args.args.index(name)
            from_end = index - len(valid_model_fit_args.args)
            default = None
            if from_end < len(valid_model_fit_args.defaults):
                default = valid_model_fit_args.defaults[from_end]
            fit_args[name] = _cast_attr(value, default)
        elif name in valid_estimator_fit_args.args:
            index = valid_estimator_fit_args.args.index(name)
            from_end = index - len(valid_estimator_fit_args.args)
            default = None
            if from_end < len(valid_estimator_fit_args.defaults):
                default = valid_estimator_fit_args.defaults[from_end]
            fit_args[name] = _cast_attr(value, default)
        else:
            unknown_args.append(name)

    unknown_args += unpaired

    if unknown_args:
        msg = ansi.bold("Valid model attributes") + ": %s\n" % ', '.join(sorted(model_attrs.keys()))
        msg += ansi.bold("Valid estimator attributes") + ": %s\n" % ', '.join(sorted(estimator_attrs.keys()))
        msg += ansi.bold("Valid pipeline attributes") + ": %s\n" % ', '.join(sorted(pipeline_attrs.keys()))
        msg += ansi.bold("Valid fit arguments") + ": %s\n" % ', '.join(sorted(valid_fit_args))

        sys.exit(ansi.error() + ' Unknown arguments: %s\n%s' % (unknown_args, msg))

    model.fit(score=parsed.score, test=parsed.test, **fit_args)
    print(ansi.success() + ' Fitting: %i\n%s' % (model.fitting, json.dumps(model.stats, indent=2)))
Esempio n. 5
0
def install_python_version():
    if env.launched():
        return

    if not env.PYTHON_VERSION:
        env.set_python_version('.'.join(sys.version_info))
        print(ansi.warning() + ' %s does not exist. Creating with %s' %
              (env.VERSION_PATH, env.PYTHON_VERSION))
        with open(env.VERSION_PATH, 'w', encoding='utf-8') as f:
            f.write(env.PYTHON_VERSION + os.linesep)

    if platform.system() == 'Windows':
        print(ansi.warning() + ' Lore only uses the installed python version on Windows.')
    else:
        if not env.PYENV:
            sys.exit(
                ansi.error() + ' pyenv is not installed. Lore is broken. try:\n'
                               ' $ pip uninstall lore && pip install lore\n'
            )

        versions = subprocess.check_output(
            (env.BIN_PYENV, 'versions', '--bare')
        ).decode('utf-8').split(os.linesep)
        if env.PYTHON_VERSION not in versions:
            print(ansi.success('INSTALL') + ' python %s' % env.PYTHON_VERSION)
            if platform.system() == 'Darwin':
                install_xcode()
            subprocess.check_call(('git', '-C', env.PYENV, 'pull'))
            subprocess.check_call((env.BIN_PYENV, 'install', env.PYTHON_VERSION))
            subprocess.check_call((env.BIN_PYENV, 'rehash'))
Esempio n. 6
0
def install_jupyter_kernel():
    if os.path.exists(env.jupyter_kernel_path):
        return

    print(ansi.success('INSTALL') + ' jupyter kernel')
    subprocess.check_call((env.bin_python, '-m', 'ipykernel', 'install',
                           '--user', '--name=' + env.project))
Esempio n. 7
0
def create_virtual_env():
    if env.PYENV:
        try:
            os.unlink(os.path.join(env.PYENV, 'versions', env.APP))
        except OSError as e:
            pass

    if os.path.exists(env.BIN_PYTHON):
        return

    print(ansi.success('CREATE') + ' virtualenv: %s' % env.APP)
    if platform.system() == 'Windows':
        subprocess.check_call((
            sys.executable,
            '-m',
            'venv',
            env.PREFIX
        ))
    else:
        subprocess.check_call((
            env.BIN_PYENV,
            'virtualenv',
            env.PYTHON_VERSION,
            env.APP
        ))

    subprocess.check_call((env.BIN_PYTHON, '-m', 'pip', 'install', '--upgrade', 'pip'))
Esempio n. 8
0
def install_bazel():
    if which('bazel'):
        return

    install_homebrew()
    print(ansi.success('INSTALL') + ' bazel for tensorflow')
    subprocess.check_call(('brew', 'install', 'bazel'))
Esempio n. 9
0
def create_virtual_env():
    if env.pyenv:
        try:
            os.unlink(os.path.join(env.pyenv, 'versions', env.project))
        except OSError as e:
            pass
    
    if os.path.exists(env.bin_python):
        return
    
    print(ansi.success('CREATE') + ' virtualenv: %s' % env.project)
    if platform.system() == 'Windows':
        subprocess.check_call((
            sys.executable,
            '-m',
            'venv',
            env.prefix
        ))
    else:
        subprocess.check_call((
            env.bin_pyenv,
            'virtualenv',
            env.python_version,
            env.project
        ))
Esempio n. 10
0
def install_gcc_5():
    if which('gcc-5'):
        return

    install_homebrew()
    print(ansi.success('INSTALL') + ' gcc 5 for xgboost')
    subprocess.check_call(('brew', 'install', 'gcc@5'))
Esempio n. 11
0
def install_python_version():
    if env.launched():
        return

    if not env.python_version:
        env.set_python_version('.'.join(sys.version_info))
        print(ansi.warning() + ' %s does not exist. Creating with %s' %
              (env.version_path, env.python_version))
        with open(env.version_path, 'w', encoding='utf-8') as f:
            f.write(env.python_version + os.linesep)

    if not env.pyenv:
        sys.exit(ansi.error() +
                 ' pyenv is not installed. Lore is broken. try:\n'
                 ' $ pip uninstall lore && pip install lore\n')

    versions = subprocess.check_output(
        (env.bin_pyenv, 'versions',
         '--bare')).decode('utf-8').split(os.linesep)
    if env.python_version not in versions:
        print(ansi.success('INSTALL') + ' python %s' % env.python_version)
        if platform.system() == 'Darwin':
            install_xcode()
        subprocess.check_call(('git', '-C', env.pyenv, 'pull'))
        subprocess.check_call((env.bin_pyenv, 'install', env.python_version))
        subprocess.check_call((env.bin_pyenv, 'rehash'))
Esempio n. 12
0
def install_pyenv():
    home = os.environ.get('HOME')
    if not home:
        return
    
    pyenv = os.path.join(home, '.pyenv')
    bin_pyenv = os.path.join(pyenv, 'bin', 'pyenv')
    virtualenv = os.path.join(pyenv, 'plugins', 'pyenv-virtualenv')
    if os.path.exists(bin_pyenv) and os.path.exists(virtualenv):
        return
    
    if os.path.exists(pyenv) and not os.path.isfile(bin_pyenv):
        print(ansi.warning() + ' pyenv executable is not present at %s' % bin_pyenv)
        while True:
            answer = input('Would you like to blow away ~/.pyenv and rebuild from scratch? [Y/n] ')
            if answer in ['', 'y', 'Y']:
                shutil.rmtree(pyenv)
                break
            elif answer in ['n', 'N']:
                sys.exit(ansi.error() + ' please fix pyenv before continuing')
            else:
                print('please enter Y or N')
    
    if not os.path.exists(pyenv):
        print(ansi.success('INSTALLING') + ' pyenv')
        subprocess.check_call((
            'git',
            'clone',
            'https://github.com/pyenv/pyenv.git',
            pyenv
        ))
    else:
        print(ansi.success('CHECK') + ' existing pyenv installation')
    env.pyenv = pyenv
    env.bin_pyenv = bin_pyenv
    env.set_python_version(env.python_version)
    
    if not os.path.exists(virtualenv):
        print(ansi.success('INSTALLING') + ' pyenv virtualenv')
        subprocess.check_call((
            'git',
            'clone',
            'https://github.com/pyenv/pyenv-virtualenv.git',
            virtualenv
        ))
    else:
        print(ansi.success('CHECK') + ' existing virtualenv installation')
Esempio n. 13
0
def install_homebrew():
    if which('brew'):
        return

    print(ansi.success('INSTALL') + ' homebrew')
    subprocess.check_call((
        '/usr/bin/ruby', '-e',
        '"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"'
    ))
Esempio n. 14
0
def install_graphviz():
    install_homebrew()
    try:
        if subprocess.check_output(('brew', 'ls', '--versions', 'graphviz')):
            return
    except:
        pass
    print(ansi.success('INSTALL') + ' graphviz')
    subprocess.check_call(('brew', 'install', 'graphviz'))
Esempio n. 15
0
def execute(parsed, unknown):
    if len(unknown) == 0:
        print(ansi.error() + ' no args to execute!')
        return

    print(ansi.success('EXECUTE ') + ' '.join(unknown))

    os.environ['PATH'] = os.path.join(env.PREFIX, 'bin') + ':' + os.environ['PATH']
    subprocess.check_call(unknown, env=os.environ)
Esempio n. 16
0
def install_jupyter_kernel():
    env.require(lore.dependencies.JUPYTER)
    if not os.path.exists(env.BIN_JUPYTER):
        return

    if env.JUPYTER_KERNEL_PATH and os.path.exists(env.JUPYTER_KERNEL_PATH):
        return

    print(ansi.success('INSTALL') + ' jupyter kernel')
    subprocess.check_call((env.BIN_PYTHON, '-m', 'ipykernel', 'install',
                           '--user', '--name=' + env.APP))
Esempio n. 17
0
def test(parsed, unknown):
    with timer('boot time'):
        if 'LORE_ENV' not in os.environ:
            env.name = env.TEST
            logger.level = logging.WARN

        import unittest
        if parsed.modules:
            names = parsed.modules.split(',')
            print(ansi.success('RUNNING ') + 'tests in ' + str(names))
            suite = unittest.TestLoader().loadTestsFromNames(names)
        else:
            print(ansi.success('RUNNING ') + 'all tests')
            suite = unittest.defaultTestLoader.discover(env.tests_dir)

    result = unittest.TextTestRunner().run(suite)
    if not result.wasSuccessful():
        sys.exit(1)
    else:
        sys.exit(0)
Esempio n. 18
0
def console(parsed, unknown):
    install_jupyter_kernel()
    sys.argv[0] = env.bin_jupyter
    args = [env.bin_jupyter, 'console', '--kernel', env.project] + unknown
    startup = '.ipython'
    if not os.path.exists(startup):
        with open(startup, 'w+') as file:
            file.write('import lore\n')

    print(ansi.success('JUPYTER') + ' ' + str(env.bin_jupyter))
    os.environ['PYTHONSTARTUP'] = startup
    os.execv(env.bin_jupyter, args)
Esempio n. 19
0
def create_virtual_env():
    if env.pyenv:
        try:
            os.unlink(os.path.join(env.pyenv, 'versions', env.project))
        except OSError as e:
            pass

    if os.path.exists(env.bin_python):
        return

    print(ansi.success('CREATE') + ' virtualenv: %s' % env.project)
    subprocess.check_call(
        (env.bin_pyenv, 'virtualenv', env.python_version, env.project))
Esempio n. 20
0
def pip_install(path, args):
    if not os.path.exists(path):
        return

    pip_args = [env.bin_python, '-m', 'pip', 'install', '-r', path]
    if hasattr(args, 'upgrade') and args.upgrade:
        pip_args += ['--upgrade', '--upgrade-strategy=eager']
    print(ansi.success('EXECUTE ') + ' '.join(pip_args))
    try:
        subprocess.check_call(pip_args)
    except subprocess.CalledProcessError:
        sys.exit(ansi.error() +
                 ' could not:\n $ lore pip install -r %s\nPlease try '
                 'installing failed packages manually, or upgrade failed '
                 'packages with:\n $ lore install --upgrade ' % path)
Esempio n. 21
0
def task(parsed, unknown):
    if len(parsed.task) == 0:
        tasks = []
        for module_finder, module_name, _ in pkgutil.iter_modules(
            [lore.env.APP + '/' + 'tasks']):
            module = importlib.import_module(lore.env.APP + '.tasks.' +
                                             module_name)
            for class_name, member in inspect.getmembers(module):
                if inspect.isclass(member) and issubclass(
                        member, lore.tasks.base.Base) and hasattr(
                            member, 'main'):
                    tasks.append(member)
        sys.exit('\n%s Tasks\n%s\n  %s\n' %
                 (lore.env.APP, '-' * (6 + len(lore.env.APP)), '\n  '.join(
                     '%s.%s: %s' %
                     (task.__module__, task.__name__, task.main.__doc__)
                     for task in tasks)))

    for task in parsed.task:
        task_class = _get_fully_qualified_class(task)
        instance = task_class()
        grouped, unpaired = _pair_args(unknown)
        argspec = _get_valid_fit_args(instance.main)

        defaults = [None] * (len(argspec.args) - len(argspec.defaults)) + list(
            argspec.defaults)
        valid_args = dict(zip(argspec.args, defaults))
        valid_args.pop('self', None)
        args = dict(grouped)
        unknown_args = []
        cast_args = {}
        for name, value in args.items():
            if name in valid_args:
                cast_args[name] = _cast_attr(value, valid_args[name])
            else:
                unknown_args.append(name)
        unknown_args += unpaired

        if unknown_args:
            msg = ansi.bold("Valid task arguments") + ": \n%s\n" % "\n".join(
                '  %s=%s' % i for i in valid_args.items())
            sys.exit(ansi.error() + ' Unknown arguments: %s\n%s\n%s' %
                     (unknown_args, msg, instance.main.__doc__))

        with timer('execute %s' % task):
            print(ansi.success('RUNNING ') + task)
            logger.info('starting task: %s %s' % (task, args))
            instance.main(**cast_args)
Esempio n. 22
0
def _generate_template(type, parsed, **kwargs):
    env.require(lore.dependencies.INFLECTION)
    import inflection
    name = parsed.name
    kwargs = kwargs or {}
    for attr in ['keras', 'xgboost', 'sklearn']:
        if hasattr(parsed, attr):
            kwargs[attr] = getattr(parsed, attr)
    kwargs['major_version'] = sys.version_info[0]
    kwargs['full_version'] = env.PYTHON_VERSION
    notebooks = ['features', 'architecture']
    name = inflection.underscore(name)
    if type == 'notebooks':
        for notebook in notebooks:
            _generate_template(notebook, parsed, **kwargs)
        return

    if type == 'test':
        destination = os.path.join(inflection.pluralize(type), 'unit',
                                   'test_' + name + '.py')
    elif type in notebooks:
        destination = os.path.join('notebooks', name, type + '.ipynb')
    else:
        destination = os.path.join(env.APP, inflection.pluralize(type),
                                   name + '.py')

    if os.path.exists(destination):
        sys.exit(ansi.error() + ' %s already exists' % destination)

    dir = os.path.dirname(destination)
    if not os.path.exists(dir):
        os.makedirs(dir)
        if type not in notebooks:
            open(os.path.join(dir, '__init__.py'), 'w')

    kwargs['app_name'] = env.APP
    kwargs['module_name'] = name
    kwargs['class_name'] = inflection.camelize(name)
    code = _render_template(type + '.py.j2', **kwargs)

    with open(destination, 'w+') as file:
        file.write(code)

    print(ansi.success('CREATED ') + destination)
Esempio n. 23
0
def _generate_template(type, parsed, **kwargs):
    import inflection
    name = parsed.name
    kwargs = kwargs or {}
    kwargs['keras'] = parsed.keras
    kwargs['xgboost'] = parsed.xgboost
    kwargs['sklearn'] = parsed.sklearn
    kwargs['major_version'] = sys.version_info[0]
    kwargs['full_version'] = lore.env.python_version
    notebooks = ['features', 'architecture']
    name = inflection.underscore(name)
    if type == 'notebooks':
        for notebook in notebooks:
            _generate_template(notebook, parsed, **kwargs)
        return

    if type == 'test':
        destination = os.path.join(inflection.pluralize(type), 'unit',
                                   'test_' + name + '.py')
    elif type in notebooks:
        destination = os.path.join('notebooks', name, type + '.ipynb')
    else:
        destination = os.path.join(lore.env.project,
                                   inflection.pluralize(type), name + '.py')

    if os.path.exists(destination):
        sys.exit(ansi.error() + ' %s already exists' % destination)

    dir = os.path.dirname(destination)
    if not os.path.exists(dir):
        os.makedirs(dir)
        if type not in notebooks:
            open(os.path.join(dir, '__init__.py'), 'w')

    kwargs['app_name'] = lore.env.project
    kwargs['module_name'] = name
    kwargs['class_name'] = inflection.camelize(name)
    code = _render_template(type + '.py.j2', **kwargs)

    with open(destination, 'w+') as file:
        file.write(code)

    print(ansi.success('CREATED ') + destination)
Esempio n. 24
0
def freeze_requirements():
    source = env.requirements

    print(
        ansi.success('EXECUTE') + ' ' + env.bin_python + ' -m pip freeze -r ' +
        source)
    vcs = split_vcs_lines()
    pip = subprocess.Popen(
        (env.bin_python, '-m', 'pip', 'freeze', '-r', source),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    (stdout, stderr) = pip.communicate()
    pip.wait()

    restore_vcs_lines(vcs)

    present = stdout.decode('utf-8').split(os.linesep)
    errors = stderr.decode('utf-8').split(os.linesep)
    missing = [line for line in errors if 'package is not installed' in line]
    regex = re.compile(r'contains ([\w\-\_]+)')
    needed = [
        m.group(1).lower() for l in missing for m in [regex.search(l)] if m
    ]

    added_index = present.index(
        '## The following requirements were added by pip freeze:')
    unsafe = None
    if added_index:
        added = present[added_index + 1:-1]
        present = set(present[0:added_index])
        safe = set()
        unsafe = set()
        for package in added:
            name = package.split('==')[0]

            for bad in vcs:
                if name in bad:
                    unsafe.add(package)
                    continue

            if name.lower() in needed:
                needed.remove(name.lower())

            safe.add(package)
        present |= safe
        present -= unsafe

    if needed:
        args = [env.bin_python, '-m', 'pip', 'install'] + needed
        print(ansi.success('EXECUTE ') + ' '.join(args))
        subprocess.check_call(args)
        return freeze_requirements()

    if unsafe:
        if vcs:
            print(ansi.warning() +
                  ' Non pypi packages were detected in your ' +
                  ansi.underline('requirements.txt') + ' that can not be '
                  'completely frozen by pip. ' + os.linesep + os.linesep +
                  os.linesep.join(vcs))

        print(ansi.info() + ' You should check the following packages in to ' +
              ansi.underline('requirements.txt') +
              ' or `lore pip uninstall` them')
        if vcs:
            print(
                ansi.warning('unless') +
                ' they are covered by the previously listed packages'
                ' that pip can not freeze.')
        print(os.linesep + os.linesep.join(unsafe) + os.linesep)

    with open(source, 'w', encoding='utf-8') as f:
        f.write(
            os.linesep.join(sorted(present, key=lambda s: s.lower())).strip() +
            os.linesep)
        if vcs:
            f.write(os.linesep.join(vcs) + os.linesep)
Esempio n. 25
0
def pip(parsed, unknown):
    args = [env.BIN_PYTHON, '-m', 'pip'] + unknown
    print(ansi.success('EXECUTE ') + ' '.join(args))
    subprocess.check_call(args)
Esempio n. 26
0
def notebook(parsed, unknown):
    install_jupyter_kernel()
    args = [env.BIN_JUPYTER, 'notebook'] + unknown
    print(ansi.success('JUPYTER') + ' ' + str(env.BIN_JUPYTER))
    os.execv(env.BIN_JUPYTER, args)
Esempio n. 27
0
def build_tensorflow(version):
    install_bazel()
    print(ansi.success('BUILD') + ' tensorflow for this architecture')

    tensorflow_repo = os.path.join(env.pyenv, 'cache', 'tensorflow')
    cache = os.path.join(env.pyenv, 'cache', 'tensorflow_pkg')
    if not os.path.exists(tensorflow_repo):
        subprocess.check_call(
            ('git', 'clone', 'https://github.com/tensorflow/tensorflow',
             tensorflow_repo))

    subprocess.check_call(('git', 'checkout', '--', '.'), cwd=tensorflow_repo)
    subprocess.check_call(('git', 'checkout', 'master'), cwd=tensorflow_repo)
    subprocess.check_call(('git', 'pull'), cwd=tensorflow_repo)
    subprocess.check_call(('git', 'checkout', 'v' + version),
                          cwd=tensorflow_repo)
    major, minor, patch = env.python_version.split('.')
    lib = os.path.join('lib', 'python' + major + '.' + minor, 'site-packages')
    new_env = {
        'PATH': os.environ['PATH'],
        'PYTHON_BIN_PATH': env.bin_python,
        'PYTHON_LIB_PATH': os.path.join(env.prefix, lib),
        'TF_NEED_MKL': '0',
        'CC_OPT_FLAGS': '-march=native -O2',
        'TF_NEED_JEMALLOC': '1',  # only available on linux regardless
        'TF_NEED_GCP': '0',
        'TF_NEED_HDFS': '0',
        'TF_ENABLE_XLA': '0',
        'TF_NEED_VERBS': '0',
        'TF_NEED_OPENCL': '0',
        'TF_NEED_S3': '0',
        'TF_NEED_GDR': '0',
        'TF_NEED_CUDA': '0',  # TODO enable CUDA when appropriate
        'TF_CUDA_CLANG': '1',
        'TF_CUDA_VERSION': '8.0.61',
        'CUDA_TOOLKIT_PATH': '/usr/local/cuda',
        'CUDNN_INSTALL_PATH': '/usr/local/cuda',
        'TF_CUDNN_VERSION': '5.1.10',
        'TF_CUDA_CLANG': '/usr/bin/gcc',
        'TF_CUDA_COMPUTE_CAPABILITIES': '3.5,5.2',
        'TF_NEED_MPI': '0'
    }
    subprocess.check_call(('./configure', ), cwd=tensorflow_repo, env=new_env)
    # TODO remove this hack when tensorflow fixes their build
    # https://github.com/tensorflow/tensorflow/issues/12979
    pip = subprocess.Popen((
        'sed', '-i',
        "'\@https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz@d'",
        'tensorflow/workspace.bzl'),
                           stdin=None,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
    (stdout, stderr) = pip.communicate()
    pip.wait()

    subprocess.check_call(
        (
            'bazel',
            'build',
            '--config=opt',
            # '--config=cuda',  TODO enable CUDA when appropriate
            'tensorflow/tools/pip_package:build_pip_package',
        ),
        cwd=tensorflow_repo)

    subprocess.check_call(
        ('bazel-bin/tensorflow/tools/pip_package/build_pip_package', cache),
        cwd=tensorflow_repo)
Esempio n. 28
0
def hyper_fit(parsed, unknown):
    print(ansi.success('HYPER PARAM FITTING ') + parsed.model)
Esempio n. 29
0
def python(parsed, unknown):
    args = [env.bin_python] + unknown
    print(ansi.success('EXECUTE ') + ' '.join(args))
    subprocess.check_call(args)
Esempio n. 30
0
def lab(parsed, unknown):
    install_jupyter_kernel()
    args = [env.bin_jupyter, 'lab'] + unknown
    print(ansi.success('JUPYTER') + ' ' + str(env.bin_jupyter))
    os.execv(env.bin_jupyter, args)