def fit(parsed, unknown): print(ansi.success('FITTING ') + parsed.model) Model = _get_fully_qualified_class(parsed.model) model = Model() valid_model_fit_args = _get_valid_fit_args(model.fit) valid_estimator_fit_args = _get_valid_fit_args(model.estimator.fit) valid_fit_args = valid_model_fit_args.args[1:] + valid_estimator_fit_args.args[1:] model_attrs = _filter_private_attributes(model.__dict__) pipeline_attrs = _filter_private_attributes(model.pipeline.__dict__) estimator_attrs = _filter_private_attributes(model.estimator.__dict__) estimator_attrs.pop('model', None) grouped, unpaired = _pair_args(unknown) # assign args to their receivers fit_args = {} unknown_args = [] for name, value in grouped: if name in model_attrs: value = _cast_attr(value, getattr(model, name)) setattr(model, name, value) elif name in pipeline_attrs: value = _cast_attr(value, getattr(model.pipeline, name)) setattr(model.pipeline, name, value) elif name in estimator_attrs: value = _cast_attr(value, getattr(model.estimator, name)) setattr(model.estimator, name, value) elif name in valid_model_fit_args.args: index = valid_model_fit_args.args.index(name) from_end = index - len(valid_model_fit_args.args) default = None if from_end < len(valid_model_fit_args.defaults): default = valid_model_fit_args.defaults[from_end] fit_args[name] = _cast_attr(value, default) elif name in valid_estimator_fit_args.args: index = valid_estimator_fit_args.args.index(name) from_end = index - len(valid_estimator_fit_args.args) default = None if from_end < len(valid_estimator_fit_args.defaults): default = valid_estimator_fit_args.defaults[from_end] fit_args[name] = _cast_attr(value, default) else: unknown_args.append(name) unknown_args += unpaired if unknown_args: msg = ansi.bold("Valid model attributes") + ": %s\n" % ', '.join(sorted(model_attrs.keys())) msg += ansi.bold("Valid estimator attributes") + ": %s\n" % ', '.join(sorted(estimator_attrs.keys())) msg += ansi.bold("Valid pipeline attributes") + ": %s\n" % ', '.join(sorted(pipeline_attrs.keys())) msg += ansi.bold("Valid fit arguments") + ": %s\n" % ', '.join(sorted(valid_fit_args)) sys.exit(ansi.error() + ' Unknown arguments: %s\n%s' % (unknown_args, msg)) model.fit(score=parsed.score, test=parsed.test, **fit_args) print(ansi.success() + ' Fitting: %i\n%s' % (model.fitting, json.dumps(model.stats, indent=2)))
def validate(): """Display error messages and exit if no lore environment can be found. """ if not os.path.exists(os.path.join(ROOT, APP, '__init__.py')): message = ansi.error() + ' Python module not found.' if os.environ.get('LORE_APP') is None: message += ' $LORE_APP is not set. Should it be different than "%s"?' % APP else: message += ' $LORE_APP is set to "%s". Should it be different?' % APP sys.exit(message) if exists(): return if len(sys.argv) > 1: command = sys.argv[1] else: command = 'lore' sys.exit( ansi.error() + ' %s is only available in lore ' 'app directories (missing %s)' % ( ansi.bold(command), ansi.underline(VERSION_PATH) ) )
def timer(message="elapsed time:", level=logging.INFO, logger=None, librato=True): global _librato, _nested_timers, _previous_timer_level, _ascii_pipes, _timer_logger if logger is None: logger = _timer_logger if level < logger.level: yield return _nested_timers += 1 start = datetime.now() try: yield finally: time = datetime.now() - start if librato and _librato and level >= logging.INFO: librato_name = 'timer.' + message.lower() librato_name = librato_name.split(':')[0] librato_name = re.sub(r'[^A-Za-z0-9\.\-_]', '', librato_name)[0:255] librato_record(librato_name, time.total_seconds()) _nested_timers -= 1 if _nested_timers == 0 or not env.unicode_locale: _ascii_pipes = '' else: delta = (_nested_timers - _previous_timer_level) length = _nested_timers * 2 if delta < 0: _ascii_pipes = _ascii_pipes[0:length] join = '┌' if _ascii_pipes[-2] == ' ' else '├' _ascii_pipes = _ascii_pipes[0:-2] + join + '─' else: _ascii_pipes = re.sub(r'[├┌]', '│', _ascii_pipes).replace('─', ' ') if delta == 0: _ascii_pipes = _ascii_pipes[:-2] + '├─' else: gap = length - len(_ascii_pipes) - 2 _ascii_pipes = _ascii_pipes + ' ' * gap + '┌─' _previous_timer_level = _nested_timers logger.log( level, (ansi.bold(_ascii_pipes + '[' + str(time) + '] ') + message))
def task(parsed, unknown): if len(parsed.task) == 0: tasks = [] for module_finder, module_name, _ in pkgutil.iter_modules( [lore.env.APP + '/' + 'tasks']): module = importlib.import_module(lore.env.APP + '.tasks.' + module_name) for class_name, member in inspect.getmembers(module): if inspect.isclass(member) and issubclass( member, lore.tasks.base.Base) and hasattr( member, 'main'): tasks.append(member) sys.exit('\n%s Tasks\n%s\n %s\n' % (lore.env.APP, '-' * (6 + len(lore.env.APP)), '\n '.join( '%s.%s: %s' % (task.__module__, task.__name__, task.main.__doc__) for task in tasks))) for task in parsed.task: task_class = _get_fully_qualified_class(task) instance = task_class() grouped, unpaired = _pair_args(unknown) argspec = _get_valid_fit_args(instance.main) defaults = [None] * (len(argspec.args) - len(argspec.defaults)) + list( argspec.defaults) valid_args = dict(zip(argspec.args, defaults)) valid_args.pop('self', None) args = dict(grouped) unknown_args = [] cast_args = {} for name, value in args.items(): if name in valid_args: cast_args[name] = _cast_attr(value, valid_args[name]) else: unknown_args.append(name) unknown_args += unpaired if unknown_args: msg = ansi.bold("Valid task arguments") + ": \n%s\n" % "\n".join( ' %s=%s' % i for i in valid_args.items()) sys.exit(ansi.error() + ' Unknown arguments: %s\n%s\n%s' % (unknown_args, msg, instance.main.__doc__)) with timer('execute %s' % task): print(ansi.success('RUNNING ') + task) logger.info('starting task: %s %s' % (task, args)) instance.main(**cast_args)
def validate(): if not os.path.exists(os.path.join(root, project, '__init__.py')): sys.exit( ansi.error() + ' Python module not found. Do you need to change $LORE_PROJECT from "%s"?' % project) if exists(): return if len(sys.argv) > 1: command = sys.argv[1] else: command = 'lore' sys.exit(ansi.error() + ' %s is only available in lore ' 'project directories (missing %s)' % (ansi.bold(command), ansi.underline(version_path)))