def main(argv=None):
parser = create_parser()
args = parser.parse_args(argv)
init_logging(args.debug)
try:
transformer = Transformer(args.path)
except (TransformerSchemaException, IOError) as e:
logging.warn('Invalid feature model: %s' % e.message)
print_exception(e)
return INVALID_TRANSFORMER_CONFIG
try:
if args.input is not None:
file_format = os.path.splitext(args.input)[1][1:]
with open(args.input, 'r') as train_fp:
transformer.train(
streamingiterload(train_fp, source_format=file_format))
elif args.extraction is not None:
train_context = list_to_dict(args.train_params)
try:
plan = ExtractionPlan(args.extraction)
train_handler = ImportHandler(plan, train_context)
except ImportHandlerException, e:
logging.warn('Invalid extraction plan: %s' % e.message)
print_exception(e)
return INVALID_EXTRACTION_PLAN
logging.info('Starting training with params:')
for key, value in train_context.items():
logging.info('%s --> %s' % (key, value))
transformer.train(train_handler)
else:
def main(argv=None):
parser = create_parser()
args = parser.parse_args(argv)
init_logging(args.debug)
try:
if args.user_params is not None:
param_list = [x.split('=', 1) for x in args.user_params]
context = dict((key, value) for (key, value) in param_list)
else:
context = {}
logging.info('User-defined parameters:')
for key, value in context.items():
logging.info('%s --> %s' % (key, value))
try:
plan = ExtractionPlan(args.path)
extractor = ImportHandler(plan, context)
except ImportHandlerException, e:
logging.warn('Invalid extraction plan: {}'.format(e.message))
print_exception(e)
return INVALID_EXTRACTION_PLAN
if args.output is not None:
logging.info('Storing data to %s...' % args.output)
getattr(extractor, 'store_data_{}'.format(args.format),
extractor.store_data_json)(args.output)
logging.info('Total %s lines' % (extractor.count, ))
logging.info('Ignored %s lines' % (extractor.ignored, ))
def main(argv=None):
parser = create_parser()
args = parser.parse_args(argv)
init_logging(args.debug)
try:
model = FeatureModel(args.path)
except IOError, exc:
logging.warn("Can't load features file. {0!s}".format(exc))
print_exception(exc)
return INVALID_FEATURE_MODEL
def main(argv=None):
parser = create_parser()
args = parser.parse_args(argv)
init_logging(args.debug)
try:
with open(args.path, 'r') as fp:
trainer = load_trainer(fp)
except (IOError, InvalidTrainerFile) as exc:
logging.warn('Invalid trainer file: {0!s}'.format(exc))
print_exception(exc)
return INVALID_TRAINER
try:
iterator = None
if args.input is not None:
# Read evaluation data from file.
eval_fp = open(args.input, 'r')
file_format = determine_data_format(args.input)
iterator = streamingiterload(eval_fp, source_format=file_format)
elif args.extraction is not None:
# Use import handler
try:
eval_context = list_to_dict(args.eval_params)
plan = ExtractionPlan(args.extraction)
eval_handler = ImportHandler(plan, eval_context)
except ImportHandlerException, e:
logging.warn('Invalid extraction plan: %s' % e.message)
print_exception(e)
return INVALID_EXTRACTION_PLAN
logging.info('Starting training with params:')
for key, value in eval_context.items():
logging.info('%s --> %s' % (key, value))
iterator = eval_handler
else:
parser.print_help()
return PARAMETERS_REQUIRED
eval_method = EVALUATION_METHODS.get(args.method)
if eval_method is not None:
eval_method(iterator, trainer, list_to_dict(args.params))
if args.input is not None:
eval_fp.close()
if args.store_vect is not None:
logging.info('Storing vectorized data to %s' % args.store_vect)
trainer.store_vect_data(trainer.predict_data.values(),
args.store_vect)
except Exception as e:
logging.info('Error occurred during prediction: %s' % e.message)
print_exception(e)
return PREDICTION_ERROR
return DONE
def create_parser():
""" Setups argument parser """
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from cloudml.trainer import __version__
program_version = 'v%s' % __version__
program_version_message = '%%(prog)s %s ' % (program_version, )
program_shortdesc = __import__('__main__').__doc__
parser = ArgumentParser(description=program_shortdesc,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('-V',