def __init__(self, _class, **kwargs):
if isinstance(_class, str):
_class = util.get_attr(_class)
Step.__init__(self, _class=_class, **kwargs)
# default name is class name
if hasattr(_class, '__name__'):
self.name = _class.__name__
def drake_step(basedir, params, method, inputs=[], tagdir=None, preview=False):
d = params_dir(basedir, params, method)
if not os.path.exists(d) and not preview:
os.makedirs(d)
dirname = os.path.join(d, 'output/')
params_file = os.path.join(d, 'params.yaml')
if params_new(params, params_file) and not preview:
with open(params_file, 'w') as f:
yaml.dump(params, f)
if tagdir is not None and not os.path.exists(tagdir) and not preview:
os.symlink(d, tagdir)
cls = util.get_attr(params[method]['name'])
if hasattr(cls, 'DEPENDENCIES'):
inputs = inputs + cls.DEPENDENCIES
inputs = ', !' + str.join(', !', inputs) if len(inputs) > 0 else ''
return '!'+dirname + ' <- ' + '!'+params_file + inputs + ' [method:' + method + ']\n\n'
parser = argparse.ArgumentParser(description='Use this script to run a single model.')
parser.add_argument('params', type=str, help='yaml params filename')
parser.add_argument('outputdir', type=str, help='output directory')
parser.add_argument('datadir', type=str, default=None, help='datadir filename')
args = parser.parse_args()
with open(args.params) as f:
params = yaml.load(f)
data_name = params['data'].pop('name')
model_name = params['model'].pop('name')
print 'Reading ' + data_name + ' with parameters:'
print pformat_indent(params['data'], 4)
data = util.get_attr(data_name)(**params['data'])
data.read(args.datadir)
print 'Tranforming with parameters:'
print pformat_indent(params['transform'], 4)
data.transform(**params['transform'])
train,test = data.cv
print 'Training ' + model_name
print ' with parameters ' + str(params['model'])
print ' on ' + str(train.sum()) + ' examples'
print ' with ' + str(len(data.X.columns)) + ' features'
estimator = util.get_attr(model_name)(**params['model'])
import yaml
import os
import argparse
from copy import deepcopy
from drain import util
import warnings
parser = argparse.ArgumentParser(description='Use this script to read and write ModelData for caching.')
parser.add_argument('input', type=str, help='yaml filename')
parser.add_argument('basedir', type=str, help='directory to cache in')
args = parser.parse_args()
with open(args.input) as f:
params_orig = yaml.load(f)
params = deepcopy(params_orig)
data_name = params['data'].pop('name')
print 'Loading ' + data_name
print ' with parameters ' + str(params['data'])
data = util.get_attr(data_name)(**params['data'])
data.read()
if not os.path.exists(args.basedir):
os.makedirs(args.basedir)
data.write(args.basedir)
def step_multi_constructor(loader, tag_suffix, node):
cls = util.get_attr(tag_suffix[1:])
kwargs = loader.construct_mapping(node, deep=True)
return cls(**kwargs)
def step_multi_constructor(loader, tag_suffix, node):
cls = util.get_attr(tag_suffix[1:])
kwargs = loader.construct_mapping(node)
return StepTemplate(_cls=cls, **kwargs)
def step_multi_constructor(loader, tag_suffix, node):
cls = util.get_attr(tag_suffix[1:])
kwargs = loader.construct_mapping(node, deep=True)
return cls(**kwargs)
def run(self, **update_kwargs):
kwargs = self.get_arguments(inputs=False, inputs_mapping=False)
kwargs.update(update_kwargs)
cls = util.get_attr(kwargs.pop('__class_name__'))
return cls(**kwargs)