def install_xcode(): result = subprocess.call(('xcode-select', '--install'), stderr=subprocess.PIPE) if result > 0: print(ansi.success('CHECK') + ' xcode-select --install') else: print(ansi.success('INSTALL') + ' xcode command line tools')
def install_pyenv(): virtualenv = os.path.join(env.PYENV, 'plugins', 'pyenv-virtualenv') if os.path.exists(env.BIN_PYENV) and os.path.exists(virtualenv): return if os.path.exists(env.PYENV) and not os.path.isfile(env.BIN_PYENV): print(ansi.warning() + ' pyenv executable is not present at %s' % env.BIN_PYENV) while True: answer = input( 'Would you like to blow away ~/.pyenv and rebuild from scratch? [Y/n] ' ) if answer in ['', 'y', 'Y']: shutil.rmtree(lore.env.PYENV) break elif answer in ['n', 'N']: sys.exit(ansi.error() + ' please fix pyenv before continuing') else: print('please enter Y or N') if not os.path.exists(env.PYENV): print(ansi.success('INSTALLING') + ' pyenv') subprocess.check_call( ('git', 'clone', 'https://github.com/pyenv/pyenv.git', env.PYENV)) else: print(ansi.success('CHECK') + ' existing pyenv installation') env.set_python_version(env.PYTHON_VERSION) if not os.path.exists(virtualenv): print(ansi.success('INSTALLING') + ' pyenv virtualenv') subprocess.check_call( ('git', 'clone', 'https://github.com/pyenv/pyenv-virtualenv.git', virtualenv)) else: print(ansi.success('CHECK') + ' existing virtualenv installation')
def install_tensorflow(): description = subprocess.check_output( (env.bin_python, '-m', 'pip', 'show', 'tensorflow')).decode('utf-8') version = re.match('.*^Version: ([^\n]+)', description, re.S | re.M).group(1) if not version: sys.exit(ansi.error() + ' tensorflow is not in requirements.txt') print(ansi.success('NATIVE') + ' tensorflow ' + version) python_version = ''.join(env.python_version.split('.')[0:2]) cached = os.path.join( env.pyenv, 'cache', 'tensorflow_pkg', 'tensorflow-' + version + '-cp' + python_version + '*') paths = glob.glob(cached) if not paths: build_tensorflow(version) paths = glob.glob(cached) path = paths[0] subprocess.check_call( (env.bin_python, '-m', 'pip', 'uninstall', '-y', 'tensorflow')) print(ansi.success('INSTALL') + ' tensorflow native build') subprocess.check_call((env.bin_python, '-m', 'pip', 'install', path))
def fit(parsed, unknown): print(ansi.success('FITTING ') + parsed.model) Model = _get_fully_qualified_class(parsed.model) model = Model() valid_model_fit_args = _get_valid_fit_args(model.fit) valid_estimator_fit_args = _get_valid_fit_args(model.estimator.fit) valid_fit_args = valid_model_fit_args.args[1:] + valid_estimator_fit_args.args[1:] model_attrs = _filter_private_attributes(model.__dict__) pipeline_attrs = _filter_private_attributes(model.pipeline.__dict__) estimator_attrs = _filter_private_attributes(model.estimator.__dict__) estimator_attrs.pop('model', None) grouped, unpaired = _pair_args(unknown) # assign args to their receivers fit_args = {} unknown_args = [] for name, value in grouped: if name in model_attrs: value = _cast_attr(value, getattr(model, name)) setattr(model, name, value) elif name in pipeline_attrs: value = _cast_attr(value, getattr(model.pipeline, name)) setattr(model.pipeline, name, value) elif name in estimator_attrs: value = _cast_attr(value, getattr(model.estimator, name)) setattr(model.estimator, name, value) elif name in valid_model_fit_args.args: index = valid_model_fit_args.args.index(name) from_end = index - len(valid_model_fit_args.args) default = None if from_end < len(valid_model_fit_args.defaults): default = valid_model_fit_args.defaults[from_end] fit_args[name] = _cast_attr(value, default) elif name in valid_estimator_fit_args.args: index = valid_estimator_fit_args.args.index(name) from_end = index - len(valid_estimator_fit_args.args) default = None if from_end < len(valid_estimator_fit_args.defaults): default = valid_estimator_fit_args.defaults[from_end] fit_args[name] = _cast_attr(value, default) else: unknown_args.append(name) unknown_args += unpaired if unknown_args: msg = ansi.bold("Valid model attributes") + ": %s\n" % ', '.join(sorted(model_attrs.keys())) msg += ansi.bold("Valid estimator attributes") + ": %s\n" % ', '.join(sorted(estimator_attrs.keys())) msg += ansi.bold("Valid pipeline attributes") + ": %s\n" % ', '.join(sorted(pipeline_attrs.keys())) msg += ansi.bold("Valid fit arguments") + ": %s\n" % ', '.join(sorted(valid_fit_args)) sys.exit(ansi.error() + ' Unknown arguments: %s\n%s' % (unknown_args, msg)) model.fit(score=parsed.score, test=parsed.test, **fit_args) print(ansi.success() + ' Fitting: %i\n%s' % (model.fitting, json.dumps(model.stats, indent=2)))
def install_python_version(): if env.launched(): return if not env.PYTHON_VERSION: env.set_python_version('.'.join(sys.version_info)) print(ansi.warning() + ' %s does not exist. Creating with %s' % (env.VERSION_PATH, env.PYTHON_VERSION)) with open(env.VERSION_PATH, 'w', encoding='utf-8') as f: f.write(env.PYTHON_VERSION + os.linesep) if platform.system() == 'Windows': print(ansi.warning() + ' Lore only uses the installed python version on Windows.') else: if not env.PYENV: sys.exit( ansi.error() + ' pyenv is not installed. Lore is broken. try:\n' ' $ pip uninstall lore && pip install lore\n' ) versions = subprocess.check_output( (env.BIN_PYENV, 'versions', '--bare') ).decode('utf-8').split(os.linesep) if env.PYTHON_VERSION not in versions: print(ansi.success('INSTALL') + ' python %s' % env.PYTHON_VERSION) if platform.system() == 'Darwin': install_xcode() subprocess.check_call(('git', '-C', env.PYENV, 'pull')) subprocess.check_call((env.BIN_PYENV, 'install', env.PYTHON_VERSION)) subprocess.check_call((env.BIN_PYENV, 'rehash'))
def install_jupyter_kernel(): if os.path.exists(env.jupyter_kernel_path): return print(ansi.success('INSTALL') + ' jupyter kernel') subprocess.check_call((env.bin_python, '-m', 'ipykernel', 'install', '--user', '--name=' + env.project))
def create_virtual_env(): if env.PYENV: try: os.unlink(os.path.join(env.PYENV, 'versions', env.APP)) except OSError as e: pass if os.path.exists(env.BIN_PYTHON): return print(ansi.success('CREATE') + ' virtualenv: %s' % env.APP) if platform.system() == 'Windows': subprocess.check_call(( sys.executable, '-m', 'venv', env.PREFIX )) else: subprocess.check_call(( env.BIN_PYENV, 'virtualenv', env.PYTHON_VERSION, env.APP )) subprocess.check_call((env.BIN_PYTHON, '-m', 'pip', 'install', '--upgrade', 'pip'))
def install_bazel(): if which('bazel'): return install_homebrew() print(ansi.success('INSTALL') + ' bazel for tensorflow') subprocess.check_call(('brew', 'install', 'bazel'))
def create_virtual_env(): if env.pyenv: try: os.unlink(os.path.join(env.pyenv, 'versions', env.project)) except OSError as e: pass if os.path.exists(env.bin_python): return print(ansi.success('CREATE') + ' virtualenv: %s' % env.project) if platform.system() == 'Windows': subprocess.check_call(( sys.executable, '-m', 'venv', env.prefix )) else: subprocess.check_call(( env.bin_pyenv, 'virtualenv', env.python_version, env.project ))
def install_gcc_5(): if which('gcc-5'): return install_homebrew() print(ansi.success('INSTALL') + ' gcc 5 for xgboost') subprocess.check_call(('brew', 'install', 'gcc@5'))
def install_python_version(): if env.launched(): return if not env.python_version: env.set_python_version('.'.join(sys.version_info)) print(ansi.warning() + ' %s does not exist. Creating with %s' % (env.version_path, env.python_version)) with open(env.version_path, 'w', encoding='utf-8') as f: f.write(env.python_version + os.linesep) if not env.pyenv: sys.exit(ansi.error() + ' pyenv is not installed. Lore is broken. try:\n' ' $ pip uninstall lore && pip install lore\n') versions = subprocess.check_output( (env.bin_pyenv, 'versions', '--bare')).decode('utf-8').split(os.linesep) if env.python_version not in versions: print(ansi.success('INSTALL') + ' python %s' % env.python_version) if platform.system() == 'Darwin': install_xcode() subprocess.check_call(('git', '-C', env.pyenv, 'pull')) subprocess.check_call((env.bin_pyenv, 'install', env.python_version)) subprocess.check_call((env.bin_pyenv, 'rehash'))
def install_pyenv(): home = os.environ.get('HOME') if not home: return pyenv = os.path.join(home, '.pyenv') bin_pyenv = os.path.join(pyenv, 'bin', 'pyenv') virtualenv = os.path.join(pyenv, 'plugins', 'pyenv-virtualenv') if os.path.exists(bin_pyenv) and os.path.exists(virtualenv): return if os.path.exists(pyenv) and not os.path.isfile(bin_pyenv): print(ansi.warning() + ' pyenv executable is not present at %s' % bin_pyenv) while True: answer = input('Would you like to blow away ~/.pyenv and rebuild from scratch? [Y/n] ') if answer in ['', 'y', 'Y']: shutil.rmtree(pyenv) break elif answer in ['n', 'N']: sys.exit(ansi.error() + ' please fix pyenv before continuing') else: print('please enter Y or N') if not os.path.exists(pyenv): print(ansi.success('INSTALLING') + ' pyenv') subprocess.check_call(( 'git', 'clone', 'https://github.com/pyenv/pyenv.git', pyenv )) else: print(ansi.success('CHECK') + ' existing pyenv installation') env.pyenv = pyenv env.bin_pyenv = bin_pyenv env.set_python_version(env.python_version) if not os.path.exists(virtualenv): print(ansi.success('INSTALLING') + ' pyenv virtualenv') subprocess.check_call(( 'git', 'clone', 'https://github.com/pyenv/pyenv-virtualenv.git', virtualenv )) else: print(ansi.success('CHECK') + ' existing virtualenv installation')
def install_homebrew(): if which('brew'): return print(ansi.success('INSTALL') + ' homebrew') subprocess.check_call(( '/usr/bin/ruby', '-e', '"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"' ))
def install_graphviz(): install_homebrew() try: if subprocess.check_output(('brew', 'ls', '--versions', 'graphviz')): return except: pass print(ansi.success('INSTALL') + ' graphviz') subprocess.check_call(('brew', 'install', 'graphviz'))
def execute(parsed, unknown): if len(unknown) == 0: print(ansi.error() + ' no args to execute!') return print(ansi.success('EXECUTE ') + ' '.join(unknown)) os.environ['PATH'] = os.path.join(env.PREFIX, 'bin') + ':' + os.environ['PATH'] subprocess.check_call(unknown, env=os.environ)
def install_jupyter_kernel(): env.require(lore.dependencies.JUPYTER) if not os.path.exists(env.BIN_JUPYTER): return if env.JUPYTER_KERNEL_PATH and os.path.exists(env.JUPYTER_KERNEL_PATH): return print(ansi.success('INSTALL') + ' jupyter kernel') subprocess.check_call((env.BIN_PYTHON, '-m', 'ipykernel', 'install', '--user', '--name=' + env.APP))
def test(parsed, unknown): with timer('boot time'): if 'LORE_ENV' not in os.environ: env.name = env.TEST logger.level = logging.WARN import unittest if parsed.modules: names = parsed.modules.split(',') print(ansi.success('RUNNING ') + 'tests in ' + str(names)) suite = unittest.TestLoader().loadTestsFromNames(names) else: print(ansi.success('RUNNING ') + 'all tests') suite = unittest.defaultTestLoader.discover(env.tests_dir) result = unittest.TextTestRunner().run(suite) if not result.wasSuccessful(): sys.exit(1) else: sys.exit(0)
def console(parsed, unknown): install_jupyter_kernel() sys.argv[0] = env.bin_jupyter args = [env.bin_jupyter, 'console', '--kernel', env.project] + unknown startup = '.ipython' if not os.path.exists(startup): with open(startup, 'w+') as file: file.write('import lore\n') print(ansi.success('JUPYTER') + ' ' + str(env.bin_jupyter)) os.environ['PYTHONSTARTUP'] = startup os.execv(env.bin_jupyter, args)
def create_virtual_env(): if env.pyenv: try: os.unlink(os.path.join(env.pyenv, 'versions', env.project)) except OSError as e: pass if os.path.exists(env.bin_python): return print(ansi.success('CREATE') + ' virtualenv: %s' % env.project) subprocess.check_call( (env.bin_pyenv, 'virtualenv', env.python_version, env.project))
def pip_install(path, args): if not os.path.exists(path): return pip_args = [env.bin_python, '-m', 'pip', 'install', '-r', path] if hasattr(args, 'upgrade') and args.upgrade: pip_args += ['--upgrade', '--upgrade-strategy=eager'] print(ansi.success('EXECUTE ') + ' '.join(pip_args)) try: subprocess.check_call(pip_args) except subprocess.CalledProcessError: sys.exit(ansi.error() + ' could not:\n $ lore pip install -r %s\nPlease try ' 'installing failed packages manually, or upgrade failed ' 'packages with:\n $ lore install --upgrade ' % path)
def task(parsed, unknown): if len(parsed.task) == 0: tasks = [] for module_finder, module_name, _ in pkgutil.iter_modules( [lore.env.APP + '/' + 'tasks']): module = importlib.import_module(lore.env.APP + '.tasks.' + module_name) for class_name, member in inspect.getmembers(module): if inspect.isclass(member) and issubclass( member, lore.tasks.base.Base) and hasattr( member, 'main'): tasks.append(member) sys.exit('\n%s Tasks\n%s\n %s\n' % (lore.env.APP, '-' * (6 + len(lore.env.APP)), '\n '.join( '%s.%s: %s' % (task.__module__, task.__name__, task.main.__doc__) for task in tasks))) for task in parsed.task: task_class = _get_fully_qualified_class(task) instance = task_class() grouped, unpaired = _pair_args(unknown) argspec = _get_valid_fit_args(instance.main) defaults = [None] * (len(argspec.args) - len(argspec.defaults)) + list( argspec.defaults) valid_args = dict(zip(argspec.args, defaults)) valid_args.pop('self', None) args = dict(grouped) unknown_args = [] cast_args = {} for name, value in args.items(): if name in valid_args: cast_args[name] = _cast_attr(value, valid_args[name]) else: unknown_args.append(name) unknown_args += unpaired if unknown_args: msg = ansi.bold("Valid task arguments") + ": \n%s\n" % "\n".join( ' %s=%s' % i for i in valid_args.items()) sys.exit(ansi.error() + ' Unknown arguments: %s\n%s\n%s' % (unknown_args, msg, instance.main.__doc__)) with timer('execute %s' % task): print(ansi.success('RUNNING ') + task) logger.info('starting task: %s %s' % (task, args)) instance.main(**cast_args)
def _generate_template(type, parsed, **kwargs): env.require(lore.dependencies.INFLECTION) import inflection name = parsed.name kwargs = kwargs or {} for attr in ['keras', 'xgboost', 'sklearn']: if hasattr(parsed, attr): kwargs[attr] = getattr(parsed, attr) kwargs['major_version'] = sys.version_info[0] kwargs['full_version'] = env.PYTHON_VERSION notebooks = ['features', 'architecture'] name = inflection.underscore(name) if type == 'notebooks': for notebook in notebooks: _generate_template(notebook, parsed, **kwargs) return if type == 'test': destination = os.path.join(inflection.pluralize(type), 'unit', 'test_' + name + '.py') elif type in notebooks: destination = os.path.join('notebooks', name, type + '.ipynb') else: destination = os.path.join(env.APP, inflection.pluralize(type), name + '.py') if os.path.exists(destination): sys.exit(ansi.error() + ' %s already exists' % destination) dir = os.path.dirname(destination) if not os.path.exists(dir): os.makedirs(dir) if type not in notebooks: open(os.path.join(dir, '__init__.py'), 'w') kwargs['app_name'] = env.APP kwargs['module_name'] = name kwargs['class_name'] = inflection.camelize(name) code = _render_template(type + '.py.j2', **kwargs) with open(destination, 'w+') as file: file.write(code) print(ansi.success('CREATED ') + destination)
def _generate_template(type, parsed, **kwargs): import inflection name = parsed.name kwargs = kwargs or {} kwargs['keras'] = parsed.keras kwargs['xgboost'] = parsed.xgboost kwargs['sklearn'] = parsed.sklearn kwargs['major_version'] = sys.version_info[0] kwargs['full_version'] = lore.env.python_version notebooks = ['features', 'architecture'] name = inflection.underscore(name) if type == 'notebooks': for notebook in notebooks: _generate_template(notebook, parsed, **kwargs) return if type == 'test': destination = os.path.join(inflection.pluralize(type), 'unit', 'test_' + name + '.py') elif type in notebooks: destination = os.path.join('notebooks', name, type + '.ipynb') else: destination = os.path.join(lore.env.project, inflection.pluralize(type), name + '.py') if os.path.exists(destination): sys.exit(ansi.error() + ' %s already exists' % destination) dir = os.path.dirname(destination) if not os.path.exists(dir): os.makedirs(dir) if type not in notebooks: open(os.path.join(dir, '__init__.py'), 'w') kwargs['app_name'] = lore.env.project kwargs['module_name'] = name kwargs['class_name'] = inflection.camelize(name) code = _render_template(type + '.py.j2', **kwargs) with open(destination, 'w+') as file: file.write(code) print(ansi.success('CREATED ') + destination)
def freeze_requirements(): source = env.requirements print( ansi.success('EXECUTE') + ' ' + env.bin_python + ' -m pip freeze -r ' + source) vcs = split_vcs_lines() pip = subprocess.Popen( (env.bin_python, '-m', 'pip', 'freeze', '-r', source), stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = pip.communicate() pip.wait() restore_vcs_lines(vcs) present = stdout.decode('utf-8').split(os.linesep) errors = stderr.decode('utf-8').split(os.linesep) missing = [line for line in errors if 'package is not installed' in line] regex = re.compile(r'contains ([\w\-\_]+)') needed = [ m.group(1).lower() for l in missing for m in [regex.search(l)] if m ] added_index = present.index( '## The following requirements were added by pip freeze:') unsafe = None if added_index: added = present[added_index + 1:-1] present = set(present[0:added_index]) safe = set() unsafe = set() for package in added: name = package.split('==')[0] for bad in vcs: if name in bad: unsafe.add(package) continue if name.lower() in needed: needed.remove(name.lower()) safe.add(package) present |= safe present -= unsafe if needed: args = [env.bin_python, '-m', 'pip', 'install'] + needed print(ansi.success('EXECUTE ') + ' '.join(args)) subprocess.check_call(args) return freeze_requirements() if unsafe: if vcs: print(ansi.warning() + ' Non pypi packages were detected in your ' + ansi.underline('requirements.txt') + ' that can not be ' 'completely frozen by pip. ' + os.linesep + os.linesep + os.linesep.join(vcs)) print(ansi.info() + ' You should check the following packages in to ' + ansi.underline('requirements.txt') + ' or `lore pip uninstall` them') if vcs: print( ansi.warning('unless') + ' they are covered by the previously listed packages' ' that pip can not freeze.') print(os.linesep + os.linesep.join(unsafe) + os.linesep) with open(source, 'w', encoding='utf-8') as f: f.write( os.linesep.join(sorted(present, key=lambda s: s.lower())).strip() + os.linesep) if vcs: f.write(os.linesep.join(vcs) + os.linesep)
def pip(parsed, unknown): args = [env.BIN_PYTHON, '-m', 'pip'] + unknown print(ansi.success('EXECUTE ') + ' '.join(args)) subprocess.check_call(args)
def notebook(parsed, unknown): install_jupyter_kernel() args = [env.BIN_JUPYTER, 'notebook'] + unknown print(ansi.success('JUPYTER') + ' ' + str(env.BIN_JUPYTER)) os.execv(env.BIN_JUPYTER, args)
def build_tensorflow(version): install_bazel() print(ansi.success('BUILD') + ' tensorflow for this architecture') tensorflow_repo = os.path.join(env.pyenv, 'cache', 'tensorflow') cache = os.path.join(env.pyenv, 'cache', 'tensorflow_pkg') if not os.path.exists(tensorflow_repo): subprocess.check_call( ('git', 'clone', 'https://github.com/tensorflow/tensorflow', tensorflow_repo)) subprocess.check_call(('git', 'checkout', '--', '.'), cwd=tensorflow_repo) subprocess.check_call(('git', 'checkout', 'master'), cwd=tensorflow_repo) subprocess.check_call(('git', 'pull'), cwd=tensorflow_repo) subprocess.check_call(('git', 'checkout', 'v' + version), cwd=tensorflow_repo) major, minor, patch = env.python_version.split('.') lib = os.path.join('lib', 'python' + major + '.' + minor, 'site-packages') new_env = { 'PATH': os.environ['PATH'], 'PYTHON_BIN_PATH': env.bin_python, 'PYTHON_LIB_PATH': os.path.join(env.prefix, lib), 'TF_NEED_MKL': '0', 'CC_OPT_FLAGS': '-march=native -O2', 'TF_NEED_JEMALLOC': '1', # only available on linux regardless 'TF_NEED_GCP': '0', 'TF_NEED_HDFS': '0', 'TF_ENABLE_XLA': '0', 'TF_NEED_VERBS': '0', 'TF_NEED_OPENCL': '0', 'TF_NEED_S3': '0', 'TF_NEED_GDR': '0', 'TF_NEED_CUDA': '0', # TODO enable CUDA when appropriate 'TF_CUDA_CLANG': '1', 'TF_CUDA_VERSION': '8.0.61', 'CUDA_TOOLKIT_PATH': '/usr/local/cuda', 'CUDNN_INSTALL_PATH': '/usr/local/cuda', 'TF_CUDNN_VERSION': '5.1.10', 'TF_CUDA_CLANG': '/usr/bin/gcc', 'TF_CUDA_COMPUTE_CAPABILITIES': '3.5,5.2', 'TF_NEED_MPI': '0' } subprocess.check_call(('./configure', ), cwd=tensorflow_repo, env=new_env) # TODO remove this hack when tensorflow fixes their build # https://github.com/tensorflow/tensorflow/issues/12979 pip = subprocess.Popen(( 'sed', '-i', "'\@https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz@d'", 'tensorflow/workspace.bzl'), stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = pip.communicate() pip.wait() subprocess.check_call( ( 'bazel', 'build', '--config=opt', # '--config=cuda', TODO enable CUDA when appropriate 'tensorflow/tools/pip_package:build_pip_package', ), cwd=tensorflow_repo) subprocess.check_call( ('bazel-bin/tensorflow/tools/pip_package/build_pip_package', cache), cwd=tensorflow_repo)
def hyper_fit(parsed, unknown): print(ansi.success('HYPER PARAM FITTING ') + parsed.model)
def python(parsed, unknown): args = [env.bin_python] + unknown print(ansi.success('EXECUTE ') + ' '.join(args)) subprocess.check_call(args)
def lab(parsed, unknown): install_jupyter_kernel() args = [env.bin_jupyter, 'lab'] + unknown print(ansi.success('JUPYTER') + ' ' + str(env.bin_jupyter)) os.execv(env.bin_jupyter, args)