def generate_schema(class_name, schema):
"""
Generate C# declaration file for a schema.
"""
has_map = False
for field in schema["fields"]:
if field["type"].startswith("Map"): has_map = True
superclass = schema["superclass"]
if superclass == "Iced": superclass = "Object"
yield "/**"
yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_csharp.py"
yield " * Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield " */"
yield "namespace ai.h2o"
yield "{"
yield " using System;"
yield " using System.Collections.Generic;" if has_map else None
yield ""
yield " public class {name}: {super} {{".format(name=class_name, super=superclass)
for field in schema["fields"]:
if field["name"] == "__meta": continue
csharp_type = translate_type(field["type"], field["schema_name"])
yield " /// <summary>"
yield bi.wrap(field["help"], " /// ")
yield " /// </summary>"
yield " public {type} {name} {{ get; set; }}".format(type=csharp_type, name=field["name"])
yield ""
yield " }"
yield "}"
def generate_schema(class_name, schema):
"""
Generate C# declaration file for a schema.
"""
has_map = False
for field in schema["fields"]:
if field["type"].startswith("Map"): has_map = True
superclass = schema["superclass"]
if superclass == "Iced": superclass = "Object"
yield "/**"
yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_csharp.py"
yield " * Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield " */"
yield "namespace ai.h2o"
yield "{"
yield " using System;"
yield " using System.Collections.Generic;" if has_map else None
yield ""
yield " public class {name}: {super} {{".format(name=class_name,
super=superclass)
for field in schema["fields"]:
if field["name"] == "__meta": continue
csharp_type = translate_type(field["type"], field["schema_name"])
yield " /// <summary>"
yield bi.wrap(field["help"], " /// ")
yield " /// </summary>"
yield " public {type} {name} {{ get; set; }}".format(
type=csharp_type, name=field["name"])
yield ""
yield " }"
yield "}"
def gen_init(modules):
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "import inspect"
yield "import sys"
yield ""
module_strs = []
# imports estimators
for full_module, module, clz, category in sorted(modules):
if module in ["grid", "automl"]:
continue
module_strs.append('"%s"' % clz)
yield "from .%s import %s" % (module, clz)
# global methods for h2o.estimators module
yield """
module = sys.modules[__name__]
def _algo_for_estimator_(shortname, cls):
if shortname == 'H2OAutoEncoderEstimator':
return 'autoencoder'
return cls.algo
_estimator_cls_by_algo_ = {_algo_for_estimator_(name, cls): cls
for name, cls in inspect.getmembers(module, inspect.isclass)
if hasattr(cls, 'algo')}
def create_estimator(algo, **params):
if algo not in _estimator_cls_by_algo_:
raise ValueError("Unknown algo type: " + algo)
return _estimator_cls_by_algo_[algo](**params)
"""
# auto-exports
yield "__all__ = ("
yield bi.wrap('"create_estimator",', indent=" " * 4)
yield bi.wrap(", ".join(module_strs), indent=" " * 4)
yield ")"
def gen_init(modules):
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
for module, clz in modules:
yield "from .%s import %s" % (module, clz)
yield ""
yield "__all__ = ["
yield bi.wrap(", ".join('"%s"' % clz for _, clz in modules), indent=" ")
yield "]"
def gen_init(modules):
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
for module, clz in modules:
yield "from .%s import %s" % (module, clz)
yield ""
yield "__all__ = ["
yield bi.wrap(", ".join('"%s"' % clz for _, clz in modules), indent=" ")
yield "]"
def get_help(param, indent=0):
pname = param.get('name')
ptype = param.get('type')
pvalues = param.get('values')
pdefault = param.get('default_value')
phelp = param.get('help')
if not phelp:
return
if ptype == 'boolean':
phelp = "\code{Logical}. " + phelp
if pvalues:
phelp += " Must be one of: %s." % ", ".join('"%s"' % v for v in pvalues)
if pdefault is not None:
phelp += " Defaults to %s." % get_doc_default_value(param)
return bi.wrap(phelp, width=120-indent)
def gen_init(modules):
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
module_strs = []
for module, clz, category in sorted(modules):
if clz == "H2OGridSearch": continue
module_strs.append('"%s"' % clz)
yield "from .%s import %s" % (module, clz)
yield ""
yield "__all__ = ("
yield bi.wrap(", ".join(module_strs), indent=" ")
yield ")"
def generate_struct(name, schema):
bi.vprint("Generating struct " + name)
yield "struct %s {" % name
yield ""
for i, field in enumerate(schema["fields"]):
if field["name"] == "__meta": continue
thrift_type = translate_type(field["type"], field["schema_name"])
name = field["name"]
if name in thrift_reserved_words:
name += "_"
required = "required" if field["required"] else "optional"
yield bi.wrap(field["help"], indent=" # ")
yield " {num}: {req} {type} {name},".format(num=i, req=required, type=thrift_type, name=name)
yield ""
yield "}"
yield ""
def gen_init(modules):
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
module_strs = []
for module, clz, category in sorted(modules):
if clz == "H2OGridSearch": continue
module_strs.append('"%s"' % clz)
if clz == "H2OAutoML": continue
module_strs.append('"%s"' % clz)
yield "from .%s import %s" % (module, clz)
yield ""
yield "__all__ = ("
yield bi.wrap(", ".join(module_strs), indent=" ")
yield ")"
def generate_struct(name, schema):
bi.vprint("Generating struct " + name)
yield "struct %s {" % name
yield ""
for i, field in enumerate(schema["fields"]):
if field["name"] == "__meta": continue
thrift_type = translate_type(field["type"], field["schema_name"])
name = field["name"]
if name in thrift_reserved_words:
name += "_"
required = "required" if field["required"] else "optional"
yield bi.wrap(field["help"], indent=" # ")
yield " {num}: {req} {type} {name},".format(num=i,
req=required,
type=thrift_type,
name=name)
yield ""
yield "}"
yield ""
def gen_module(schema, algo):
classname = algo_to_classname(algo)
extra_imports = extra_imports_for(algo)
help_preamble = help_preamble_for(algo)
help_epilogue = help_epilogue_for(algo)
init_extra = init_extra_for(algo)
class_extra = class_extra_for(algo)
module_extra = module_extra_for(algo)
param_names = []
for param in schema["parameters"]:
assert (param["type"][:4] == "enum") == bool(param["values"]), "Values are expected for enum types only"
if param["values"]:
enum_values = [normalize_enum_constant(p) for p in param["values"]]
if param["default_value"]:
param["default_value"] = normalize_enum_constant(param["default_value"])
else:
enum_values = None
pname = param["name"]
if pname in reserved_words: pname += "_"
param_names.append(pname)
param["pname"] = pname
param["ptype"] = translate_type_for_check(param["type"], enum_values)
param["dtype"] = translate_type_for_doc(param["type"], enum_values)
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "from __future__ import absolute_import, division, print_function, unicode_literals"
yield ""
yield "from h2o.estimators.estimator_base import H2OEstimator"
yield "from h2o.exceptions import H2OValueError"
yield "from h2o.frame import H2OFrame"
if classname == "H2OStackedEnsembleEstimator":
yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric, is_type"
else:
yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric"
if extra_imports:
yield reindent_block(extra_imports, 0) + ""
yield ""
yield ""
yield "class %s(H2OEstimator):" % classname
yield " \"\"\""
yield " " + schema["algo_full_name"]
yield ""
if help_preamble:
yield " %s" % reindent_block(help_preamble, 4)
if help_epilogue:
yield ""
yield " %s" % reindent_block(help_epilogue, 4)
yield " \"\"\""
yield ""
yield ' algo = "%s"' % algo
yield ""
yield " def __init__(self, **kwargs):"
yield " super(%s, self).__init__()" % classname
yield " self._parms = {}"
yield " names_list = {%s}" % bi.wrap(", ".join('"%s"' % p for p in param_names),
indent=(" " * 22), indent_first=False)
yield ' if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")'
yield " for pname, pvalue in kwargs.items():"
yield " if pname == 'model_id':"
yield " self._id = pvalue"
yield ' self._parms["model_id"] = pvalue'
yield " elif pname in names_list:"
yield " # Using setattr(...) will invoke type-checking of the arguments"
yield " setattr(self, pname, pvalue)"
yield " else:"
yield ' raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))'
if init_extra:
yield " " + reindent_block(init_extra, 8)
yield ""
for param in schema["parameters"]:
pname = param["pname"]
ptype = param["ptype"]
if pname == "model_id": continue # The getter is already defined in ModelBase
sname = pname[:-1] if pname[-1] == '_' else pname
if param["dtype"].startswith("Enum"):
vals = param["dtype"][5:-1].split(", ")
extrahelp = "One of: " + ", ".join("``%s``" % v for v in vals)
else:
extrahelp = "Type: ``%s``" % param["dtype"]
if param["default_value"] is None:
extrahelp += "."
else:
extrahelp += " (default: ``%s``)." % stringify(param["default_value"])
yield " @property"
yield " def %s(self):" % pname
yield ' """'
yield " %s" % bi.wrap(param["help"], indent=(" " * 8), indent_first=False)
yield ""
yield " %s" % bi.wrap(extrahelp, indent=(" " * 8), indent_first=False)
yield ' """'
yield " return self._parms.get(\"%s\")" % sname
yield ""
yield " @%s.setter" % pname
yield " def %s(self, %s):" % (pname, pname)
if pname in {"initial_weights", "initial_biases"}:
yield " assert_is_type(%s, None, [H2OFrame, None])" % pname
elif pname in {"alpha", "lambda_"} and ptype == "[numeric]":
# For `alpha` and `lambda` the server reports type float[], while in practice simple floats are also ok
yield " assert_is_type(%s, None, numeric, [numeric])" % pname
elif pname in {"checkpoint", "pretrained_autoencoder"}:
yield " assert_is_type(%s, None, str, H2OEstimator)" % pname
elif pname in {"base_models"}:
yield " if is_type(base_models,[H2OEstimator]):"
yield " %s = [b.model_id for b in %s]" % (pname,pname)
yield " self._parms[\"%s\"] = %s" % (sname, pname)
yield " else:"
yield " assert_is_type(%s, None, %s)" % (pname, ptype)
yield " self._parms[\"%s\"] = %s" % (sname, pname)
else:
yield " assert_is_type(%s, None, %s)" % (pname, ptype)
if pname not in {"base_models"}:
yield " self._parms[\"%s\"] = %s" % (sname, pname)
yield ""
yield ""
if class_extra:
yield ""
yield " " + reindent_block(class_extra, 4)
if module_extra:
yield ""
yield reindent_block(module_extra, 0)
def gen_module(schema, algo):
classname = algo_to_classname(algo)
extra_imports = extra_imports_for(algo)
help_preamble = help_preamble_for(algo)
help_epilogue = help_epilogue_for(algo)
init_extra = init_extra_for(algo)
class_extra = class_extra_for(algo)
module_extra = module_extra_for(algo)
param_names = []
for param in schema["parameters"]:
assert (param["type"][:4] == "enum") == bool(param["values"]), "Values are expected for enum types only"
if param["values"]:
enum_values = [normalize_enum_constant(p) for p in param["values"]]
if param["default_value"]:
param["default_value"] = normalize_enum_constant(param["default_value"])
else:
enum_values = None
pname = param["name"]
if (pname==u'distribution') and (not(algo==u'glm') and not(algo==u'gbm')): # quasibinomial only in glm, gbm
enum_values.remove(u'quasibinomial')
if (pname==u'distribution') and (not(algo==u'glm')): # ordinal only in glm
enum_values.remove(u'ordinal')
if pname in reserved_words: pname += "_"
param_names.append(pname)
param["pname"] = pname
param["ptype"] = translate_type_for_check(param["type"], enum_values)
param["dtype"] = translate_type_for_doc(param["type"], enum_values)
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "from __future__ import absolute_import, division, print_function, unicode_literals"
yield ""
yield "from h2o.estimators.estimator_base import H2OEstimator"
yield "from h2o.exceptions import H2OValueError"
yield "from h2o.frame import H2OFrame"
if classname == "H2OStackedEnsembleEstimator":
yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric, is_type"
yield "import json"
yield "import ast"
else:
yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric"
if extra_imports:
yield reindent_block(extra_imports, 0) + ""
yield ""
yield ""
yield "class %s(H2OEstimator):" % classname
yield " \"\"\""
yield " " + schema["algo_full_name"]
yield ""
if help_preamble:
yield " %s" % reindent_block(help_preamble, 4)
if help_epilogue:
yield ""
yield " %s" % reindent_block(help_epilogue, 4)
yield " \"\"\""
yield ""
yield ' algo = "%s"' % algo
yield ""
yield " def __init__(self, **kwargs):"
yield " super(%s, self).__init__()" % classname
yield " self._parms = {}"
yield " names_list = {%s}" % bi.wrap(", ".join('"%s"' % p for p in param_names),
indent=(" " * 22), indent_first=False)
yield ' if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")'
yield " for pname, pvalue in kwargs.items():"
yield " if pname == 'model_id':"
yield " self._id = pvalue"
yield ' self._parms["model_id"] = pvalue'
if algo == 'word2vec':
yield ' elif pname == \'pre_trained\':'
yield ' setattr(self, pname, pvalue)'
yield ' self._determine_vec_size();'
yield ' setattr(self, \'vec_size\', self.vec_size)'
yield " elif pname in names_list:"
yield " # Using setattr(...) will invoke type-checking of the arguments"
yield " setattr(self, pname, pvalue)"
yield " else:"
yield ' raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))'
if init_extra:
yield " " + reindent_block(init_extra, 8)
yield ""
for param in schema["parameters"]:
pname = param["pname"]
ptype = param["ptype"]
if pname == "model_id": continue # The getter is already defined in ModelBase
sname = pname[:-1] if pname[-1] == '_' else pname
if param["dtype"].startswith("Enum"):
vals = param["dtype"][5:-1].split(", ")
extrahelp = "One of: " + ", ".join("``%s``" % v for v in vals)
else:
if pname == "metalearner_params":
extrahelp = "Type: ``dict``"
else:
extrahelp = "Type: ``%s``" % param["dtype"]
if param["default_value"] is None:
extrahelp += "."
else:
if pname == "metalearner_params":
extrahelp += " (default: ``None``)."
else:
extrahelp += " (default: ``%s``)." % stringify(param["default_value"])
if (pname == "offset_column" or pname == "distribution") and algo == "drf":
yield " @property"
yield " def %s(self):" % pname
yield ' """'
yield " [Deprecated] %s" % bi.wrap(param["help"], indent=(" " * 8), indent_first=False)
yield ""
yield " %s" % bi.wrap(extrahelp, indent=(" " * 8), indent_first=False)
else:
yield " @property"
yield " def %s(self):" % pname
yield ' """'
yield " %s" % bi.wrap(param["help"], indent=(" " * 8), indent_first=False)
yield ""
yield " %s" % bi.wrap(extrahelp, indent=(" " * 8), indent_first=False)
if pname == "metalearner_params":
yield " Example: metalearner_gbm_params = {'max_depth': 2, 'col_sample_rate': 0.3}"
yield ' """'
if pname != "metalearner_params":
yield " return self._parms.get(\"%s\")" % sname
else:
yield " if self._parms.get(\"%s\") != None:" % sname
yield " metalearner_params_dict = ast.literal_eval(self._parms.get(\"%s\"))" % sname
yield " for k in metalearner_params_dict:"
yield " if len(metalearner_params_dict[k]) == 1: #single parameter"
yield " metalearner_params_dict[k] = metalearner_params_dict[k][0]"
yield " return metalearner_params_dict"
yield " else:"
yield " return self._parms.get(\"%s\")" % sname
yield ""
yield " @%s.setter" % pname
yield " def %s(self, %s):" % (pname, pname)
if pname in {"initial_weights", "initial_biases"}:
yield " assert_is_type(%s, None, [H2OFrame, None])" % pname
elif pname in {"alpha", "lambda_"} and ptype == "[numeric]":
# For `alpha` and `lambda` the server reports type float[], while in practice simple floats are also ok
yield " assert_is_type(%s, None, numeric, [numeric])" % pname
elif pname in {"checkpoint", "pretrained_autoencoder"}:
yield " assert_is_type(%s, None, str, H2OEstimator)" % pname
elif pname in {"base_models"}:
yield " if is_type(base_models,[H2OEstimator]):"
yield " %s = [b.model_id for b in %s]" % (pname,pname)
yield " self._parms[\"%s\"] = %s" % (sname, pname)
yield " else:"
yield " assert_is_type(%s, None, %s)" % (pname, ptype)
yield " self._parms[\"%s\"] = %s" % (sname, pname)
elif pname in {"metalearner_params"}:
yield " assert_is_type(%s, None, %s)" % (pname, "dict")
yield ' if %s is not None and %s != "":' % (pname, pname)
yield " for k in %s:" % (pname)
yield ' if ("[" and "]") not in str(metalearner_params[k]):'
yield " metalearner_params[k]=[metalearner_params[k]]"
yield " self._parms[\"%s\"] = str(json.dumps(%s))" % (sname, pname)
yield " else:"
yield " self._parms[\"%s\"] = None" % (sname)
else:
yield " assert_is_type(%s, None, %s)" % (pname, ptype)
if pname not in {"base_models", "metalearner_params"}:
yield " self._parms[\"%s\"] = %s" % (sname, pname)
yield ""
yield ""
if class_extra:
yield ""
yield " " + reindent_block(class_extra, 4)
if module_extra:
yield ""
yield reindent_block(module_extra, 0)
def gen_module(schema, algo):
classname = algo_to_classname(algo)
extra_imports = extra_imports_for(algo)
help_preamble = help_preamble_for(algo)
help_epilogue = help_epilogue_for(algo)
init_extra = init_extra_for(algo)
class_extra = class_extra_for(algo)
module_extra = module_extra_for(algo)
param_names = []
for param in schema["parameters"]:
assert (param["type"][:4] == "enum") == bool(
param["values"]), "Values are expected for enum types only"
if param["values"]:
enum_values = [normalize_enum_constant(p) for p in param["values"]]
if param["default_value"]:
param["default_value"] = normalize_enum_constant(
param["default_value"])
else:
enum_values = None
pname = param["name"]
if (pname == u'distribution') and (
not (algo == u'glm')
and not (algo == u'gbm')): # quasibinomial only in glm, gbm
enum_values.remove(u'quasibinomial')
if (pname == u'distribution') and (
not (algo == u'glm')): # ordinal only in glm
enum_values.remove(u'ordinal')
if (pname == u'distribution') and (
not (algo == u'gbm')): # custom only in gbm
enum_values.remove(u'custom')
if (pname == u'stopping_metric') and (
not (algo == u'isolationforest')
): # anomaly_score only in Isolation Forest
enum_values.remove(u'anomaly_score')
if (pname == u'stopping_metric') and (algo == u'isolationforest'):
enum_values = [u'AUTO', u'anomaly_score']
if pname in reserved_words: pname += "_"
param_names.append(pname)
param["pname"] = pname
param["ptype"] = translate_type_for_check(param["type"], enum_values)
param["dtype"] = translate_type_for_doc(param["type"], enum_values)
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "from __future__ import absolute_import, division, print_function, unicode_literals"
yield ""
yield "from h2o.estimators.estimator_base import H2OEstimator"
yield "from h2o.exceptions import H2OValueError"
yield "from h2o.frame import H2OFrame"
if classname == "H2OStackedEnsembleEstimator":
yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric, is_type"
yield "import json"
yield "import ast"
else:
yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric"
if extra_imports:
yield reindent_block(extra_imports, 0) + ""
yield ""
yield ""
yield "class %s(H2OEstimator):" % classname
yield " \"\"\""
yield " " + schema["algo_full_name"]
yield ""
if help_preamble:
yield " %s" % reindent_block(help_preamble, 4)
if help_epilogue:
yield ""
yield " %s" % reindent_block(help_epilogue, 4)
yield " \"\"\""
yield ""
yield ' algo = "%s"' % algo
yield ""
yield " def __init__(self, **kwargs):"
yield " super(%s, self).__init__()" % classname
yield " self._parms = {}"
yield " names_list = {%s}" % bi.wrap(", ".join(
'"%s"' % p for p in param_names),
indent=(" " * 22),
indent_first=False)
yield ' if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")'
yield " for pname, pvalue in kwargs.items():"
yield " if pname == 'model_id':"
yield " self._id = pvalue"
yield ' self._parms["model_id"] = pvalue'
if algo == 'word2vec':
yield ' elif pname == \'pre_trained\':'
yield ' setattr(self, pname, pvalue)'
yield ' self._determine_vec_size();'
yield ' setattr(self, \'vec_size\', self.vec_size)'
yield " elif pname in names_list:"
yield " # Using setattr(...) will invoke type-checking of the arguments"
yield " setattr(self, pname, pvalue)"
yield " else:"
yield ' raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))'
if init_extra:
yield " " + reindent_block(init_extra, 8)
yield ""
for param in schema["parameters"]:
pname = param["pname"]
ptype = param["ptype"]
if pname == "model_id":
continue # The getter is already defined in ModelBase
sname = pname[:-1] if pname[-1] == '_' else pname
if param["dtype"].startswith("Enum"):
vals = param["dtype"][5:-1].split(", ")
extrahelp = "One of: " + ", ".join("``%s``" % v for v in vals)
else:
if pname == "metalearner_params":
extrahelp = "Type: ``dict``"
else:
extrahelp = "Type: ``%s``" % param["dtype"]
if param["default_value"] is None:
extrahelp += "."
else:
if pname == "metalearner_params":
extrahelp += " (default: ``None``)."
else:
extrahelp += " (default: ``%s``)." % stringify(
param["default_value"])
if (pname == "offset_column"
or pname == "distribution") and algo == "drf":
yield " @property"
yield " def %s(self):" % pname
yield ' """'
yield " [Deprecated] %s" % bi.wrap(
param["help"], indent=(" " * 8), indent_first=False)
yield ""
yield " %s" % bi.wrap(
extrahelp, indent=(" " * 8), indent_first=False)
else:
yield " @property"
yield " def %s(self):" % pname
yield ' """'
yield " %s" % bi.wrap(
param["help"], indent=(" " * 8), indent_first=False)
yield ""
yield " %s" % bi.wrap(
extrahelp, indent=(" " * 8), indent_first=False)
if pname == "metalearner_params":
yield " Example: metalearner_gbm_params = {'max_depth': 2, 'col_sample_rate': 0.3}"
yield ' """'
if pname != "metalearner_params":
yield " return self._parms.get(\"%s\")" % sname
else:
yield " if self._parms.get(\"%s\") != None:" % sname
yield " metalearner_params_dict = ast.literal_eval(self._parms.get(\"%s\"))" % sname
yield " for k in metalearner_params_dict:"
yield " if len(metalearner_params_dict[k]) == 1: #single parameter"
yield " metalearner_params_dict[k] = metalearner_params_dict[k][0]"
yield " return metalearner_params_dict"
yield " else:"
yield " return self._parms.get(\"%s\")" % sname
yield ""
yield " @%s.setter" % pname
yield " def %s(self, %s):" % (pname, pname)
if pname in {"initial_weights", "initial_biases"}:
yield " assert_is_type(%s, None, [H2OFrame, None])" % pname
elif pname in {"alpha", "lambda_"} and ptype == "[numeric]":
# For `alpha` and `lambda` the server reports type float[], while in practice simple floats are also ok
yield " assert_is_type(%s, None, numeric, [numeric])" % pname
elif pname in {"checkpoint", "pretrained_autoencoder"}:
yield " assert_is_type(%s, None, str, H2OEstimator)" % pname
elif pname in {"base_models"}:
yield " if is_type(base_models,[H2OEstimator]):"
yield " %s = [b.model_id for b in %s]" % (pname, pname)
yield " self._parms[\"%s\"] = %s" % (sname, pname)
yield " else:"
yield " assert_is_type(%s, None, %s)" % (pname, ptype)
yield " self._parms[\"%s\"] = %s" % (sname, pname)
elif pname in {"metalearner_params"}:
yield " assert_is_type(%s, None, %s)" % (pname, "dict")
yield ' if %s is not None and %s != "":' % (pname, pname)
yield " for k in %s:" % (pname)
yield ' if ("[" and "]") not in str(metalearner_params[k]):'
yield " metalearner_params[k]=[metalearner_params[k]]"
yield " self._parms[\"%s\"] = str(json.dumps(%s))" % (
sname, pname)
yield " else:"
yield " self._parms[\"%s\"] = None" % (sname)
elif ptype == "H2OFrame":
yield " self._parms[\"%s\"] = H2OFrame._validate(%s, '%s')" % (
sname, pname, pname)
else:
yield " assert_is_type(%s, None, %s)" % (pname, ptype)
if pname not in {"base_models", "metalearner_params"
} and ptype != "H2OFrame":
yield " self._parms[\"%s\"] = %s" % (sname, pname)
yield ""
yield ""
if class_extra:
yield ""
yield " " + reindent_block(class_extra, 4)
if module_extra:
yield ""
yield reindent_block(module_extra, 0)
def gen_module(schema, name):
classname = algo_to_classname(name)
extra_imports = extra_imports_for(name)
help_preamble = help_preamble_for(name)
help_epilogue = help_epilogue_for(name)
init_extra = init_extra_for(name)
class_extra = class_extra_for(name)
module_extra = module_extra_for(name)
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "from .estimator_base import H2OEstimator"
if extra_imports:
yield reindent_block(extra_imports, 0) + ""
yield ""
yield ""
yield "class %s(H2OEstimator):" % classname
yield " \"\"\""
yield " " + schema["algo_full_name"]
yield " " + ("-" * len(schema["algo_full_name"]))
if help_preamble:
yield " %s" % reindent_block(help_preamble, 4)
yield ""
yield " Parameters (optional, unless specified otherwise)"
yield " ----------"
param_names = []
for param in schema["parameters"]:
assert (param["type"][:4] == "enum") == bool(param["values"]), "Values are expected for enum types only"
ptype = translate_type(param["type"], param["values"])
name = param["name"]
if name in reserved_words: name += "_"
param_names.append(name)
if param["required"]: ptype += ", required"
yield " %s : %s" % (name, bi.wrap(ptype, " " * (9 + len(name)), indent_first=False))
yield bi. wrap(param["help"], " " * 8)
if param["default_value"] is not None:
yield " Default: %s" % stringify(param["default_value"])
yield ""
if help_epilogue:
yield " %s" % reindent_block(help_epilogue, 4)
yield " \"\"\""
yield " def __init__(self, **kwargs):"
yield " super(%s, self).__init__()" % classname
yield " self._parms = {}"
yield " for name in [" + bi.wrap(", ".join('"%s"' % p for p in param_names),
indent=(" " * 21), indent_first=False) + "]:"
yield " pname = name[:-1] if name[-1] == '_' else name"
yield " self._parms[pname] = kwargs[name] if name in kwargs else None"
if init_extra:
yield " " + reindent_block(init_extra, 8)
yield ""
for param in schema["parameters"]:
name = param["name"]
if name == "model_id": continue # The getter is already defined in ModelBase
prop = name
if name in reserved_words: prop += "_"
yield " @property"
yield " def %s(self):" % prop
yield " return self._parms[\"%s\"]" % name
yield ""
yield " @%s.setter" % prop
yield " def %s(self, value):" % prop
yield " self._parms[\"%s\"] = value" % name
yield ""
if class_extra:
yield " " + reindent_block(class_extra, 4)
yield ""
if module_extra:
yield reindent_block(module_extra, 0)
yield ""
def gen_module(schema, algo):
classname = algo_to_classname(algo)
extra_imports = extra_imports_for(algo)
help_preamble = help_preamble_for(algo)
help_epilogue = help_epilogue_for(algo)
init_extra = init_extra_for(algo)
class_extra = class_extra_for(algo)
module_extra = module_extra_for(algo)
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "from .estimator_base import H2OEstimator"
if extra_imports:
yield reindent_block(extra_imports, 0) + ""
yield ""
yield ""
yield "class %s(H2OEstimator):" % classname
yield " \"\"\""
yield " " + schema["algo_full_name"]
yield ""
if help_preamble:
yield " %s" % reindent_block(help_preamble, 4)
yield ""
yield " Parameters"
yield " ----------"
param_names = []
for param in schema["parameters"]:
assert (param["type"][:4] == "enum") == bool(param["values"]), "Values are expected for enum types only"
ptype = translate_type(param["type"], param["values"])
name = param["name"]
if name in reserved_words: name += "_"
param_names.append(name)
if param["required"]: ptype += ", required"
yield " %s : %s" % (name, bi.wrap(ptype, " " * (9 + len(name)), indent_first=False))
yield bi. wrap(param["help"], " " * 8)
if param["default_value"] is not None:
yield " Default: %s" % stringify(param["default_value"])
yield ""
if help_epilogue:
yield " %s" % reindent_block(help_epilogue, 4)
yield " \"\"\""
yield ""
yield ' algo = "%s"' % algo
yield ""
yield " def __init__(self, **kwargs):"
yield " super(%s, self).__init__()" % classname
yield " self._parms = {}"
yield " for name in [" + bi.wrap(", ".join('"%s"' % p for p in param_names),
indent=(" " * 21), indent_first=False) + "]:"
yield " pname = name[:-1] if name[-1] == '_' else name"
yield " self._parms[pname] = kwargs[name] if name in kwargs else None"
if init_extra:
yield " " + reindent_block(init_extra, 8)
yield ""
for param in schema["parameters"]:
name = param["name"]
if name == "model_id": continue # The getter is already defined in ModelBase
prop = name
if name in reserved_words: prop += "_"
yield " @property"
yield " def %s(self):" % prop
yield " return self._parms[\"%s\"]" % name
yield ""
yield " @%s.setter" % prop
yield " def %s(self, value):" % prop
yield " self._parms[\"%s\"] = value" % name
yield ""
if class_extra:
yield " " + reindent_block(class_extra, 4)
yield ""
if module_extra:
yield reindent_block(module_extra, 0)
yield ""
def gen_module(schema, algo):
"""
Ideally we should be able to avoid logic specific to algos in this file.
Instead, customizations are externalized in ./python/gen_{algo}.py files.
Logic that is specific to python types (e.g. H2OFrame, enums as list...) should however stay here
as the type translation is done in this file.
"""
classname = algo_to_classname(algo)
rest_api_version = get_customizations_for(algo, 'rest_api_version')
extra_imports = get_customizations_for(algo, 'extensions.__imports__')
class_doc = get_customizations_for(algo, 'doc.__class__')
class_examples = get_customizations_for(algo, 'examples.__class__')
class_init_validation = get_customizations_for(
algo, 'extensions.__init__validation')
class_init_setparams = get_customizations_for(
algo, 'extensions.__init__setparams')
class_extras = get_customizations_for(algo, 'extensions.__class__')
module_extras = get_customizations_for(algo, 'extensions.__module__')
update_param_defaults = get_customizations_for('defaults', 'update_param')
update_param = get_customizations_for(algo, 'update_param')
def extend_schema_params(param):
pname = param.get('name')
param = deepcopy(param)
updates = None
for update_fn in [update_param, update_param_defaults]:
if callable(update_fn):
updates = update_fn(pname, param)
if updates is not None:
param = updates
break
# return param if isinstance(param, (list, tuple)) else [param] # always return array to support deprecated aliases
return param
extended_params = [extend_schema_params(p) for p in schema['parameters']]
param_names = []
for param in extended_params:
pname = param.get('name')
ptype = param.get('type')
pvalues = param.get('values')
pdefault = param.get('default_value')
assert (ptype[:4] == 'enum'
) == bool(pvalues), "Values are expected for enum types only"
if pvalues:
enum_values = [normalize_enum_constant(p) for p in pvalues]
if pdefault:
pdefault = normalize_enum_constant(pdefault)
else:
enum_values = None
if pname in reserved_words:
pname += "_"
param_names.append(pname)
param['pname'] = pname
param['default_value'] = pdefault
param['ptype'] = translate_type_for_check(ptype, enum_values)
param['dtype'] = translate_type_for_doc(ptype, enum_values)
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "from __future__ import absolute_import, division, print_function, unicode_literals"
yield ""
if extra_imports:
yield reformat_block(extra_imports)
yield "from h2o.estimators.estimator_base import H2OEstimator"
yield "from h2o.exceptions import H2OValueError"
yield "from h2o.frame import H2OFrame"
yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric"
yield ""
yield ""
yield "class %s(H2OEstimator):" % classname
yield ' """'
yield " " + schema["algo_full_name"]
yield ""
if class_doc:
yield reformat_block(class_doc, 4)
if class_examples:
yield ""
yield " :examples:"
yield ""
yield reformat_block(class_examples, 4)
yield ' """'
yield ""
yield ' algo = "%s"' % algo
yield " param_names = {%s}" % bi.wrap(", ".join('"%s"' % p
for p in param_names),
indent=(" " * 19),
indent_first=False)
yield ""
yield " def __init__(self, **kwargs):"
# TODO: generate __init__ docstring with all params (also generate exact signature to support auto-completion)
yield " super(%s, self).__init__()" % classname
yield " self._parms = {}"
if class_init_validation:
yield reformat_block(class_init_validation, 8)
yield " for pname, pvalue in kwargs.items():"
yield " if pname == 'model_id':"
yield " self._id = pvalue"
yield ' self._parms["model_id"] = pvalue'
if class_init_setparams:
yield reformat_block(class_init_setparams, 12)
yield " elif pname in self.param_names:"
yield " # Using setattr(...) will invoke type-checking of the arguments"
yield " setattr(self, pname, pvalue)"
yield " else:"
yield ' raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))'
if rest_api_version:
yield ' self._parms["_rest_version"] = %s' % rest_api_version
yield ""
for param in extended_params:
pname = param.get('pname')
if pname == "model_id":
continue # The getter is already defined in ModelBase
sname = pname[:-1] if pname[-1] == '_' else pname
ptype = param.get('ptype')
dtype = param.get('dtype')
pdefault = param.get('default_value')
if dtype.startswith("Enum"):
vals = dtype[5:-1].split(", ")
property_doc = "One of: " + ", ".join("``%s``" % v for v in vals)
else:
property_doc = "Type: ``%s``" % dtype
property_doc += ("." if pdefault is None else " (default: ``%s``)." %
stringify(pdefault))
deprecated = pname in get_customizations_for(algo, 'deprecated', [])
yield " @property"
yield " def %s(self):" % pname
yield ' """'
yield bi.wrap(
"%s%s" %
("[Deprecated] " if deprecated else "", param.get('help')),
indent=8 * ' ') # we need to wrap only for text coming from server
yield ""
yield bi.wrap(property_doc, indent=8 * ' ')
custom_property_doc = get_customizations_for(algo,
"doc.{}".format(pname))
if custom_property_doc:
yield ""
yield reformat_block(custom_property_doc, 8)
property_examples = get_customizations_for(algo,
"examples.{}".format(pname))
if property_examples:
yield ""
yield " :examples:"
yield ""
yield reformat_block(property_examples, 8)
yield ' """'
property_getter = get_customizations_for(
algo, "overrides.{}.getter".format(
pname)) # check gen_stackedensemble.py for an example
if property_getter:
yield reformat_block(property_getter.format(**locals()), 8)
else:
yield " return self._parms.get(\"%s\")" % sname
yield ""
yield " @%s.setter" % pname
yield " def %s(self, %s):" % (pname, pname)
property_setter = get_customizations_for(
algo, "overrides.{}.setter".format(
pname)) # check gen_stackedensemble.py for an example
if property_setter:
yield reformat_block(property_setter.format(**locals()), 8)
else:
# special types validation
if ptype == "H2OEstimator":
yield " assert_is_type(%s, None, str, %s)" % (pname,
ptype)
elif ptype == "H2OFrame":
yield " self._parms[\"%s\"] = H2OFrame._validate(%s, '%s')" % (
sname, pname, pname)
else:
# default validation
yield " assert_is_type(%s, None, %s)" % (pname, ptype)
if ptype != "H2OFrame":
# default assignment
yield " self._parms[\"%s\"] = %s" % (sname, pname)
yield ""
yield ""
if class_extras:
yield reformat_block(code_as_str(class_extras), 4)
if module_extras:
yield ""
yield reformat_block(code_as_str(module_extras))
def generate_schema(class_name, schema):
"""
Generate schema Java class.
:param class_name: name of the class
:param schema: information about the class
"""
superclass = schema["superclass"]
if superclass == "Schema": superclass = "Object"
has_map = False
is_model_builder = False
has_inherited = False
for field in schema["fields"]:
if field["name"] == "__meta": continue
if field["is_inherited"]:
has_inherited = True
continue
if field["type"].startswith("Map"): has_map = True
if field["name"] == "can_build": is_model_builder = True
fields = []
for field in schema["fields"]:
if field["name"] == "__meta": continue
java_type = translate_type(field["type"], field["schema_name"])
java_value = get_java_value(field)
# hackery: we flatten the parameters up into the ModelBuilder schema, rather than nesting them in the
# parameters schema class...
if False and is_model_builder and field["name"] == "parameters":
fields.append(("parameters", "null", "ModelParameterSchemaV3[]",
field["help"], field["is_inherited"]))
else:
fields.append((field["name"], java_value, java_type, field["help"],
field["is_inherited"]))
class_decl = class_name
if "generics" in schema:
class_decl += "<" + ", ".join(
"%s extends %s" % (t, long_type)
for t, long_type in schema["generics"]) + ">"
super_decl = superclass
if "super_generics" in schema:
super_decl += "<" + ", ".join(schema["super_generics"]) + ">"
yield "/*"
yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
yield " * Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield " */"
yield "package water.bindings.pojos;"
yield ""
yield "import com.google.gson.Gson;"
yield "import com.google.gson.annotations.*;"
yield "import java.util.Map;" if has_map else None
yield ""
yield ""
yield "public class %s extends %s {" % (
class_decl, super_decl) if super_decl != "Object" else None
yield "public class %s {" % (
class_decl) if super_decl == "Object" else None
yield ""
for name, value, ftype, fhelp, inherited in fields:
if inherited: continue
ccname = translate_name(name)
yield " /**"
yield bi.wrap(fhelp, indent=" * ")
yield " */"
yield " @SerializedName(\"%s\")" % name if name != ccname else None
yield " public %s %s;" % (ftype, ccname)
yield ""
if has_inherited:
yield ""
yield " /*" + ("-" * 114)
yield " //" + (" " * 50) + "INHERITED"
yield " //" + ("-" * 114)
yield ""
for name, value, ftype, fhelp, inherited in fields:
if not inherited: continue
yield bi.wrap(fhelp, " // ")
yield " public %s %s;" % (ftype, translate_name(name))
yield ""
yield " */"
yield ""
yield " /**"
yield " * Public constructor"
yield " */"
yield " public %s() {" % class_name
for name, value, _, _, _ in fields:
if name == "parameters": continue
if value == "null": continue
yield " %s = %s;" % (translate_name(name), value)
yield " }"
yield ""
yield " /**"
yield " * Return the contents of this object as a JSON String."
yield " */"
yield " @Override"
yield " public String toString() {"
yield " return new Gson().toJson(this);"
yield " }"
yield ""
yield "}"
def generate_main_class(endpoints):
yield dedent(
0, """
/*
* This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py
* Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)
*/
package water.bindings;
import water.bindings.pojos.*;
import water.bindings.proxies.retrofit.*;
import retrofit2.*;
import retrofit2.converter.gson.GsonConverterFactory;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import okhttp3.OkHttpClient;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Type;
@SuppressWarnings("unused")
public class H2oApi {
public static String DEFAULT_URL = \"http://localhost:54321/\";
public H2oApi() {
this(DEFAULT_URL);
}
public H2oApi(String url) {
_url = url;
}
public H2oApi setUrl(String url) {
_url = url;
retrofit = null;
return this;
}
public H2oApi setTimeout(int t) {
timeout_s = t;
retrofit = null;
return this;
}
/**
* Set time interval for job polling in {@link #waitForJobCompletion(JobKeyV3)}.
* @param millis time interval, in milliseconds
*/
public H2oApi setJobPollInterval(int millis) {
pollInterval_ms = millis;
return this;
}
/**
* Continuously poll server for the status of the given job, until it completes.
* @param jobKey job to query
* @return the finished job
*/
public JobV3 waitForJobCompletion(JobKeyV3 jobKey) {
return waitForJobCompletion(keyToString(jobKey));
}
public JobV3 waitForJobCompletion(String jobId) {
Jobs jobService = getService(Jobs.class);
Response<JobsV3> jobsResponse = null;
int retries = 3;
JobsV3 jobs = null;
do {
try {
Thread.sleep(pollInterval_ms);
jobsResponse = jobService.fetch(jobId).execute();
} catch (IOException e) {
System.err.println(\"Caught exception: \" + e);
} catch (InterruptedException e) { /* pass */ }
if (jobsResponse == null || !jobsResponse.isSuccessful())
if (retries-- > 0)
continue;
else
throw new RuntimeException(\"/3/Jobs/\" + jobId + \" failed 3 times.\");
jobs = jobsResponse.body();
if (jobs.jobs == null || jobs.jobs.length != 1)
throw new RuntimeException(\"Failed to find Job: \" + jobId);
} while (jobs != null && jobs.jobs[0].status.equals(\"RUNNING\"));
return jobs == null? null : jobs.jobs[0];
}
""")
for route in endpoints:
apiname = route["api_name"]
class_name = route["class_name"]
outtype = route["output_schema"]
input_fields = route["input_params"]
required_fields = [
field for field in input_fields if field["required"]
]
input_fields_wo_excluded = [
field for field in input_fields
if field["name"] != "_exclude_fields"
]
yield " /**"
yield bi.wrap(route["summary"], indent=" * ")
yield " */"
# Make several versions of each API call:
# (1) Only the required parameters
# (2) All parameters except the _excluded_fields
# (3) All parameters
li = len(input_fields)
le = len(input_fields_wo_excluded)
lr = len(required_fields)
assert lr <= 3, "Too many required fields in method " + apiname
if lr == li:
# The set of required fields is the same as the set of input fields. No need for (2) and (3).
input_fields = None
input_fields_wo_excluded = None
elif le == li or le == lr or li >= 4:
# If set (2) coincides with either (1) or (3), then we will not generate code for it.
# Additionally, if there are too many input params so that we will put them into a container class,
# then there will be no need for separate case (2) either.
input_fields_wo_excluded = None
# Temporary hack, since Parse requires all the fields from ParseSetup: don't generate the required-fields-only
# method
if 'parse' == apiname:
required_fields = None
for fields in [
required_fields, input_fields_wo_excluded, input_fields
]:
if fields is None: continue
use_schema_param = (len(fields) >= 4)
value_field_strs = []
typed_field_strs = []
for field in fields:
ftype = translate_type(field["type"], field["schema_name"])
fname = translate_name(field["name"])
typed_field_strs.append("%s %s" % (ftype, fname))
if use_schema_param: fname = "params." + fname
if ftype.endswith("KeyV3"):
s = "keyToString(%s)" % fname
elif ftype.endswith("KeyV3[]"):
s = "keyArrayToStringArray(%s)" % fname
elif ftype == "ColSpecifierV3":
s = "colToString(%s)" % fname
else:
s = fname
value_field_strs.append(s)
if use_schema_param:
args = route["input_schema"] + " params"
values = "\n " + ",\n ".join(
value_field_strs) + "\n "
else:
args = ", ".join(typed_field_strs)
values = ", ".join(value_field_strs)
if fields == input_fields_wo_excluded:
values += ", \"\""
method = apiname if route["handler_method"] == "exec" else route[
"handler_method"]
yield " public {type} {method}({args}) throws IOException {{".\
format(type=outtype, method=apiname, args=args)
yield " {clazz} s = getService({clazz}.class);".format(
clazz=class_name)
yield " return s.{method}({values}).execute().body();".format(
method=method, values=values)
yield " }"
yield ""
yield dedent(
2, """
//--------- PRIVATE --------------------------------------------------------------------------------------------------
private Retrofit retrofit;
private String _url = DEFAULT_URL;
private int timeout_s = 60;
private int pollInterval_ms = 1000;
private void initializeRetrofit() {
Gson gson = new GsonBuilder()
.registerTypeAdapterFactory(new ModelV3TypeAdapter())
.registerTypeAdapter(KeyV3.class, new KeySerializer())
.registerTypeAdapter(ColSpecifierV3.class, new ColSerializer())
.registerTypeAdapter(ModelBuilderSchema.class, new ModelDeserializer())
.registerTypeAdapter(ModelSchemaBaseV3.class, new ModelSchemaDeserializer())
.registerTypeAdapter(ModelOutputSchemaV3.class, new ModelOutputDeserializer())
.registerTypeAdapter(ModelParametersSchemaV3.class, new ModelParametersDeserializer())
.create();
OkHttpClient client = new OkHttpClient.Builder()
.connectTimeout(timeout_s, TimeUnit.SECONDS)
.writeTimeout(timeout_s, TimeUnit.SECONDS)
.readTimeout(timeout_s, TimeUnit.SECONDS)
.build();
this.retrofit = new Retrofit.Builder()
.client(client)
.baseUrl(_url)
.addConverterFactory(GsonConverterFactory.create(gson))
.build();
}
private Retrofit getRetrofit() {
if (retrofit == null) initializeRetrofit();
return retrofit;
}
private <T> T getService(Class<T> clazz) {
return getRetrofit().create(clazz);
}
/**
* Keys get sent as Strings and returned as objects also containing the type and URL,
* so they need a custom GSON serializer.
*/
private static class KeySerializer implements JsonSerializer<KeyV3>, JsonDeserializer<KeyV3> {
@Override
public JsonElement serialize(KeyV3 key, Type typeOfKey, JsonSerializationContext context) {
return new JsonPrimitive(key.name);
}
@Override
public KeyV3 deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) {
if (json.isJsonNull()) return null;
JsonObject jobj = json.getAsJsonObject();
String type = jobj.get(\"type\").getAsString();
switch (type) {
// TODO: dynamically generate all possible cases
case \"Key<Model>\": return context.deserialize(jobj, ModelKeyV3.class);
case \"Key<Job>\": return context.deserialize(jobj, JobKeyV3.class);
case \"Key<Grid>\": return context.deserialize(jobj, GridKeyV3.class);
case \"Key<Frame>\": return context.deserialize(jobj, FrameKeyV3.class);
default: throw new JsonParseException(\"Unable to deserialize key of type \" + type);
}
}
}
private static class ColSerializer implements JsonSerializer<ColSpecifierV3> {
@Override
public JsonElement serialize(ColSpecifierV3 col, Type typeOfCol, JsonSerializationContext context) {
return new JsonPrimitive(col.columnName);
}
}
""")
def inj(middle):
return lambda schema, algo: schema[:len(algo)] + middle + schema[len(
algo):]
for clz, base, target_clz_factory in [
("ModelDeserializer", "ModelBuilderSchema",
lambda schema, algo: schema),
("ModelSchemaDeserializer", "ModelSchemaBaseV3", inj("Model")),
("ModelOutputDeserializer", "ModelOutputSchemaV3", inj("ModelOutput")),
("ModelParametersDeserializer", "ModelParametersSchemaV3",
inj("Parameters"))
]:
yield " /**"
yield " * Factory method for parsing a %s json object into an instance of the model-specific subclass." % base
yield " */"
yield " private static class %s implements JsonDeserializer<%s> {" % (
clz, base)
yield " @Override"
yield " public %s deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)" % base
yield " throws JsonParseException {"
yield " if (json.isJsonNull()) return null;"
yield " if (json.isJsonObject()) {"
yield " JsonObject jobj = json.getAsJsonObject();"
yield " if (jobj.has(\"algo\")) {"
yield " String algo = jobj.get(\"algo\").getAsJsonPrimitive().getAsString().toLowerCase();"
yield " switch (algo) {"
for route in endpoints:
if route["class_name"] == "ModelBuilders" and route[
"api_name"].startswith("train"):
algo = route["algo"]
oschema = route["output_schema"]
assert oschema.lower()[:len(
algo)] == algo, "Wrong output schema for algo %s: %s" % (
algo, oschema)
model = target_clz_factory(oschema, algo)
yield " case \"{algo}\": return context.deserialize(json, {model}.class);".format(
**locals())
yield " default:"
yield " throw new JsonParseException(\"Unable to deserialize model of type \" + algo);"
yield " }"
yield " }"
yield " }"
yield " throw new JsonParseException(\"Invalid %s element \" + json.toString());" % base
yield " }"
yield " }"
yield ""
yield dedent(
2, """
private static class ModelV3TypeAdapter implements TypeAdapterFactory {
@Override
public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
final Class<? super T> rawType = type.getRawType();
if (!ModelBuilderSchema.class.isAssignableFrom(rawType) &&
!ModelSchemaBaseV3.class.isAssignableFrom(rawType)) return null;
final TypeAdapter<T> delegate = gson.getDelegateAdapter(this, type);
return new TypeAdapter<T>() {
@Override
public void write(JsonWriter out, T value) throws IOException {
delegate.write(out, value);
}
@Override
public T read(JsonReader in) throws IOException {
JsonObject jobj = new JsonParser().parse(in).getAsJsonObject();
if (jobj.has(\"parameters\") && jobj.get(\"parameters\").isJsonArray()) {
JsonArray jarr = jobj.get(\"parameters\").getAsJsonArray();
JsonObject paramsNew = new JsonObject();
for (JsonElement item : jarr) {
JsonObject itemObj = item.getAsJsonObject();
paramsNew.add(itemObj.get(\"name\").getAsString(), itemObj.get(\"actual_value\"));
}
jobj.add(\"parameters\", paramsNew);
}
// noinspection unchecked
return (T) new Gson().fromJson(jobj, rawType);
}
};
}
}
/**
* Return an array of Strings for an array of keys.
*/
public static String[] keyArrayToStringArray(KeyV3[] keys) {
if (keys == null) return null;
String[] ids = new String[keys.length];
int i = 0;
for (KeyV3 key : keys) ids[i++] = key.name;
return ids;
}
/**
* Return an array of keys from an array of Strings.
* @param ids array of string ids to convert to KeyV3's
* @param clz class of key objects to create. Since we have JobKeyV3, FrameKeyV3, ModelKeyV3, etc -- this
* method needs to know which of these keys you want to create
*/
public static <T extends KeyV3> T[] stringArrayToKeyArray(String[] ids, Class<T> clz) {
if (ids == null) return null;
// noinspection unchecked
T[] keys = (T[]) Array.newInstance(clz, ids.length);
String keyType = clz.getSimpleName();
if (keyType.endsWith(\"KeyV3\")) keyType = keyType.substring(0, keyType.length()-5);
try {
int i = 0;
for (String id: ids) {
keys[i] = clz.getConstructor().newInstance();
keys[i].name = id;
keys[i].type = keyType;
i++;
}
}
catch (Exception e) {
e.printStackTrace();
}
return keys;
}
/**
*
*/
public static String keyToString(KeyV3 key) {
return key == null? null : key.name;
}
/**
*
*/
public static FrameKeyV3 stringToFrameKey(String key) {
if (key == null) return null;
FrameKeyV3 k = new FrameKeyV3();
k.name = key;
return k;
}
/**
*
*/
private static String colToString(ColSpecifierV3 col) {
return col == null? null : col.columnName;
}
/**
*
*/
public static String stringToCol(String col) {
if (col == null) return null;
ColSpecifierV3 c = new ColSpecifierV3();
c.columnName = col;
return col;
}
public static void copyFields(Object to, Object from) {
Field[] fromFields = from.getClass().getDeclaredFields();
Field[] toFields = to.getClass().getDeclaredFields();
for (Field fromField : fromFields){
Field toField;
try {
toField = to.getClass().getDeclaredField(fromField.getName());
fromField.setAccessible(true);
toField.setAccessible(true);
toField.set(to, fromField.get(from));
}
catch (Exception ignored) {
// NoSuchField is the normal case
}
}
}
}
""")
def gen_module(schema, name):
classname = algo_to_classname(name)
extra_imports = extra_imports_for(name)
help_preamble = help_preamble_for(name)
help_epilogue = help_epilogue_for(name)
init_extra = init_extra_for(name)
class_extra = class_extra_for(name)
module_extra = module_extra_for(name)
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "from .estimator_base import H2OEstimator"
if extra_imports:
yield reindent_block(extra_imports, 0) + ""
yield ""
yield ""
yield "class %s(H2OEstimator):" % classname
yield " \"\"\""
yield " %s" % schema["algo_full_name"]
yield " %s" % ("-" * len(schema["algo_full_name"]))
if help_preamble:
yield " %s" % reindent_block(help_preamble, 4)
yield ""
yield " Parameters (optional, unless specified otherwise)"
yield " ----------"
for param in schema["parameters"]:
assert (param["type"] == "enum") == bool(param["values"]), "Values are expected for enum types only"
ptype = translate_type(param["type"], param["values"])
name = param["name"]
if param["required"]: ptype += ", required"
yield " %s : %s" % (name, bi.wrap(ptype, " " * (9 + len(name)), indent_first=False))
yield bi. wrap(param["help"], " " * 8)
if param["default_value"] is not None:
yield " Default: %s" % stringify(param["default_value"])
yield ""
if help_epilogue:
yield " %s" % reindent_block(help_epilogue, 4)
yield " \"\"\""
yield " def __init__(self, **kwargs):"
yield " super(%s, self).__init__()" % classname
yield " self._parms = kwargs"
if init_extra:
yield " %s" % reindent_block(init_extra, 8)
yield ""
for param in schema["parameters"]:
name = param["name"]
yield " @property"
yield " def %s(self):" % name
yield " return self._parms[\"%s\"]" % name
yield ""
yield " @%s.setter" % name
yield " def %s(self, value):" % name
yield " self._parms[\"%s\"] = value" % name
yield ""
if class_extra:
yield " %s" % reindent_block(class_extra, 4)
yield ""
if module_extra:
yield "%s" % reindent_block(module_extra, 0)
yield ""
def generate_main_class(endpoints):
yield "/**"
yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
yield " * Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield " */"
yield "package water.bindings;"
yield ""
yield "import water.bindings.pojos.*;"
yield "import water.bindings.proxies.retrofit.*;"
yield "import retrofit2.*;"
yield "import retrofit2.converter.gson.GsonConverterFactory;"
yield "import com.google.gson.*;"
yield "import okhttp3.OkHttpClient;"
yield "import java.io.IOException;"
yield "import java.lang.reflect.Type;"
yield "import java.util.concurrent.TimeUnit;"
yield ""
yield "public class H2oApi {"
yield ""
yield " public H2oApi() {}"
yield " public H2oApi(String url) { this.url = url; }"
yield ""
yield " public H2oApi setUrl(String s) {"
yield " url = s;"
yield " retrofit = null;"
yield " return this;"
yield " }"
yield ""
yield " public H2oApi setTimeout(int t) {"
yield " timeout_s = t;"
yield " retrofit = null;"
yield " return this;"
yield " }"
yield ""
yield " /**"
yield " * Set time interval for job polling in {@link #waitForJobCompletion(JobKeyV3)}."
yield " * @param millis time interval, in milliseconds"
yield " */"
yield " public H2oApi setJobPollInterval(int millis) {"
yield " pollInterval_ms = millis;"
yield " return this;"
yield " }"
yield ""
yield " /**"
yield " * Continuously poll server for the status of the given job, until it completes."
yield " * @param jobKey job to query"
yield " * @return the finished job"
yield " */"
yield " public JobV3 waitForJobCompletion(JobKeyV3 jobKey) {"
yield " return waitForJobCompletion(keyToString(jobKey));"
yield " }"
yield " public JobV3 waitForJobCompletion(String jobId) {"
yield " Jobs jobService = getService(Jobs.class);"
yield " Response<JobsV3> jobsResponse = null;"
yield " int retries = 3;"
yield " JobsV3 jobs = null;"
yield " do {"
yield " try {"
yield " Thread.sleep(pollInterval_ms);"
yield " jobsResponse = jobService.fetch(jobId).execute();"
yield " } catch (IOException e) {"
yield " System.err.println(\"Caught exception: \" + e);"
yield " } catch (InterruptedException e) { /* pass */ }"
yield " if (jobsResponse == null || !jobsResponse.isSuccessful())"
yield " if (retries-- > 0)"
yield " continue;"
yield " else"
yield " throw new RuntimeException(\"/3/Jobs/\" + jobId + \" failed 3 times.\");"
yield " jobs = jobsResponse.body();"
yield " if (jobs.jobs == null || jobs.jobs.length != 1)"
yield " throw new RuntimeException(\"Failed to find Job: \" + jobId);"
yield " } while (jobs != null && jobs.jobs[0].status.equals(\"RUNNING\"));"
yield " return jobs == null? null : jobs.jobs[0];"
yield " }"
yield ""
for route in endpoints:
apiname = route["api_name"]
class_name = route["class_name"]
outtype = route["output_schema"]
input_fields = route["input_params"]
required_fields = [
field for field in input_fields if field["required"]
]
input_fields_wo_excluded = [
field for field in input_fields
if field["name"] != "_exclude_fields"
]
yield " /**"
yield bi.wrap(route["summary"], indent=" * ")
yield " */"
# Make several versions of each API call:
# (1) Only the required parameters
# (2) All parameters except the _excluded_fields
# (3) All parameters
li = len(input_fields)
le = len(input_fields_wo_excluded)
lr = len(required_fields)
assert lr <= 3, "Too many required fields in method " + apiName
if lr == li:
# The set of required fields is the same as the set of input fields. No need for (2) and (3).
input_fields = None
input_fields_wo_excluded = None
elif le == li or le == lr or li >= 4:
# If set (2) coincides with either (1) or (3), then we will not generate code for it.
# Additionally, if there are too many input params so that we will put them into a container class,
# then there will be no need for separate case (2) either.
input_fields_wo_excluded = None
for fields in [
required_fields, input_fields_wo_excluded, input_fields
]:
if fields is None: continue
use_schema_param = (len(fields) >= 4)
value_field_strs = []
typed_field_strs = []
for field in fields:
ftype = translate_type(field["type"], field["schema_name"])
fname = translate_name(field["name"])
typed_field_strs.append("%s %s" % (ftype, fname))
if use_schema_param: fname = "params." + fname
if ftype.endswith("KeyV3"):
s = "keyToString(%s)" % fname
elif ftype.endswith("KeyV3[]"):
s = "keyArrayToStringArray(%s)" % fname
elif ftype.startswith("ColSpecifier"):
s = "colToString(%s)" % fname
else:
s = fname
value_field_strs.append(s)
if use_schema_param:
args = route["input_schema"] + " params"
values = "\n " + ",\n ".join(
value_field_strs) + "\n "
else:
args = ", ".join(typed_field_strs)
values = ", ".join(value_field_strs)
if fields == input_fields_wo_excluded:
values += ", \"\""
yield " public {type} {method}({args}) throws IOException {{".\
format(type=outtype, method=apiname, args=args)
yield " {clazz} s = getService({clazz}.class);".format(
clazz=class_name)
yield " return s.{method}({values}).execute().body();".\
format(method=route["handler_method"], values=values)
yield " }"
yield ""
yield ""
yield " //--------- PRIVATE " + "-" * 98
yield ""
yield " private Retrofit retrofit;"
yield " private String url = \"http://localhost/54321/\";"
yield " private int timeout_s = 60;"
yield " private int pollInterval_ms = 1000;"
yield ""
yield " private void initializeRetrofit() {"
yield " Gson gson = new GsonBuilder()"
yield " .registerTypeAdapter(KeyV3.class, new KeySerializer())"
yield " .registerTypeAdapter(ColSpecifierV3.class, new ColSerializer())"
yield " .registerTypeAdapter(ModelsV3.class, new ModelDeserializer())"
yield " .create();"
yield ""
yield " OkHttpClient client = new OkHttpClient.Builder()"
yield " .connectTimeout(timeout_s, TimeUnit.SECONDS)"
yield " .writeTimeout(timeout_s, TimeUnit.SECONDS)"
yield " .readTimeout(timeout_s, TimeUnit.SECONDS)"
yield " .build();"
yield ""
yield " this.retrofit = new Retrofit.Builder()"
yield " .client(client)"
yield " .baseUrl(url)"
yield " .addConverterFactory(GsonConverterFactory.create(gson))"
yield " .build();"
yield " }"
yield ""
yield " private Retrofit getRetrofit() {"
yield " if (retrofit == null) initializeRetrofit();"
yield " return retrofit;"
yield " }"
yield ""
yield " private <T> T getService(Class<T> clazz) {"
yield " return getRetrofit().create(clazz);"
yield " }"
yield ""
yield ""
yield " /**"
yield " * Keys get sent as Strings and returned as objects also containing the type and URL,"
yield " * so they need a custom GSON serializer."
yield " */"
yield " private static class KeySerializer implements JsonSerializer<KeyV3> {"
yield " @Override"
yield " public JsonElement serialize(KeyV3 key, Type typeOfKey, JsonSerializationContext context) {"
yield " return new JsonPrimitive(key.name);"
yield " }"
yield " }"
yield " private static class ColSerializer implements JsonSerializer<ColSpecifierV3> {"
yield " @Override"
yield " public JsonElement serialize(ColSpecifierV3 col, Type typeOfCol, JsonSerializationContext context) {"
yield " return new JsonPrimitive(col.columnName);"
yield " }"
yield " }"
yield " /**"
yield " * Factory method for parsing a ModelsV3 json object into an instance of the model-specific subclass."
yield " */"
yield " private static class ModelDeserializer implements JsonDeserializer<ModelsV3> {"
yield " @Override"
yield " public ModelsV3 deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)"
yield " throws JsonParseException {"
yield " if (json.isJsonNull()) return null;"
yield " if (json.isJsonObject()) {"
yield " JsonObject jobj = json.getAsJsonObject();"
yield " if (jobj.has(\"algo\")) {"
yield " String algo = jobj.get(\"algo\").getAsJsonPrimitive().getAsString().toLowerCase();"
yield " switch (algo) {"
for route in endpoints:
if route["class_name"] == "ModelBuilders" and route[
"api_name"].startswith("train"):
algo = route["algo"]
oschema = route["output_schema"]
assert oschema.lower(
)[:len(algo)] == algo, "Wrong output schema for algo %s: %s" % (
algo, oschema)
model = oschema[:len(algo)] + "Model" + oschema[len(
algo):] # "DeepLearningV3" => "DeepLearningModelV3"
yield " case \"{algo}\": return context.deserialize(json, {model}.class);".format(
**locals())
yield " default:"
yield " throw new JsonParseException(\"Unable to deserialize model of type \" + algo);"
yield " }"
yield " }"
yield " }"
yield " throw new JsonParseException(\"Invalid ModelsV3 element \" + json.toString());"
yield " }"
yield " }"
yield ""
yield " /**"
yield " * Return an array of Strings for an array of keys."
yield " */"
yield " private static String[] keyArrayToStringArray(KeyV3[] keys) {"
yield " if (keys == null) return null;"
yield " String[] ids = new String[keys.length];"
yield " int i = 0;"
yield " for (KeyV3 key : keys) ids[i++] = key.name;"
yield " return ids;"
yield " }"
yield ""
yield " /**"
yield " *"
yield " */"
yield " private static String keyToString(KeyV3 key) {"
yield " return key == null? null : key.name;"
yield " }"
yield ""
yield " /**"
yield " *"
yield " */"
yield " private static String colToString(ColSpecifierV3 col) {"
yield " return col == null? null : col.columnName;"
yield " }"
yield ""
yield "}"
def gen_module(schema, algo):
help_preamble = help_preamble_for(algo)
help_epilogue = help_epilogue_for(algo)
help_afterword = help_afterword_for(algo)
model_name = algo_to_modelname(algo)
help_example = help_example_for(algo)
help_return = help_return_for(algo)
help_references = help_references_for(algo)
class_extra = class_extra_for(algo)
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_R.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details) \n#'"
yield "# -------------------------- %s -------------------------- #" % model_name
if help_preamble:
lines = help_preamble.split("\n")
for line in lines:
yield "#' %s" % line.lstrip()
yield "#'"
yield "#' @param x A vector containing the \code{character} names of the predictors in the model."
yield "#' If x is missing,then all columns except y are used."
yield "#' @param y The name of the response variable in the model."
for param in schema["parameters"]:
if param["name"] == "ignored_columns" or param[
"name"] == "response_column":
continue
phelp = param["help"]
if param["type"] == "boolean":
phelp = "\code{Logical}. " + phelp
if param["values"]:
phelp += " Must be one of: %s." % ", ".join(
'"%s"' % p for p in param["values"])
if param["default_value"] is not None:
phelp += " Defaults to %s." % param["default_value"]
yield "#' @param %s %s" % (
param["name"],
bi.wrap(phelp, indent=("#' "), indent_first=False))
if help_return:
yield "#' @return %s" % bi.wrap(
help_return, indent=("#' "), indent_first=False)
if help_epilogue:
yield "#' @seealso %s" % bi.wrap(
help_epilogue, indent=("#' "), indent_first=False)
if help_references:
#lines = help_references.split("\n")
#for line in lines:
# yield "#' %s" % line.lstrip()
yield "#' @references %s" % bi.wrap(
help_references, indent=("#' "), indent_first=False)
if help_example:
yield "#' @examples"
lines = help_example.split("\n")
for line in lines:
yield "#' %s" % line.lstrip()
yield "#' @export"
yield "h2o.%s <- function(x, y, " % algo
list = []
for param in schema["parameters"]:
if param["name"] == "ignored_columns" or param[
"name"] == "response_column":
continue
if param["type"][:4] == "enum" or param["default_value"] is not None:
list.append(
indent("%s = %s" % (param["name"], normalize_value(param)),
17 + len(algo)))
else:
list.append(indent(param["name"], 17 + len(algo)))
yield ", \n".join(list)
yield indent(") \n{", 17 + len(algo))
yield " #If x is missing, then assume user wants to use all columns as features."
yield " if(missing(x)){"
yield " if(is.numeric(y)){"
yield " x <- setdiff(col(training_frame),y)"
yield " }else{"
yield " x <- setdiff(colnames(training_frame),y)"
yield " }"
yield " }"
yield ""
yield " # Required args: training_frame"
yield " if( missing(training_frame) ) stop(\"argument \'training_frame\' is missing, with no default\")"
yield " if( missing(validation_frame) ) validation_frame = NULL"
yield " # Training_frame and validation_frame may be a key or an H2OFrame object"
yield " if (!is.H2OFrame(training_frame))"
yield " tryCatch(training_frame <- h2o.getFrame(training_frame),"
yield " error = function(err) {"
yield " stop(\"argument \'training_frame\' must be a valid H2OFrame or key\")"
yield " })"
yield " if (!is.null(validation_frame)) {"
yield " if (!is.H2OFrame(validation_frame))"
yield " tryCatch(validation_frame <- h2o.getFrame(validation_frame),"
yield " error = function(err) {"
yield " stop(\"argument \'validation_frame\' must be a valid H2OFrame or key\")"
yield " })"
yield " }"
yield " # Parameter list to send to model builder"
yield " parms <- list()"
yield " parms$training_frame <- training_frame"
yield " args <- .verify_dataxy(training_frame, x, y, autoencoder)"
yield " if( !missing(offset_column) && !is.null(offset_column)) args$x_ignore <- args$x_ignore[!( offset_column == args$x_ignore )]"
yield " if( !missing(weights_column) && !is.null(weights_column)) args$x_ignore <- args$x_ignore[!( weights_column == args$x_ignore )]"
yield " if( !missing(fold_column) && !is.null(fold_column)) args$x_ignore <- args$x_ignore[!( fold_column == args$x_ignore )]"
yield " parms$response_column <- args$y"
yield " parms$ignored_columns <- args$x_ignore \n "
for param in schema["parameters"]:
if param["name"] == "ignored_columns" or param[
"name"] == "response_column" or param[
"name"] == "training_frame":
continue
if param["name"] == "loss":
yield " if(!missing(loss)) {"
yield " if(loss == \"MeanSquare\") {"
yield " warning(\"Loss name 'MeanSquare' is deprecated; please use 'Quadratic' instead.\")"
yield " parms$loss <- \"Quadratic\""
yield " } else "
yield " parms$loss <- loss"
yield " }"
yield " if (!missing(%s))" % param["name"]
yield " parms$%s <- %s" % (param["name"], param["name"])
yield " .h2o.modelJob('%s', parms, h2oRestApiVersion=3) \n}" % algo
if help_afterword:
lines = help_afterword.split("\n")
for line in lines:
yield "%s" % line.lstrip()
#yield "%s" % bi.wrap(help_afterword, indent=("#' "), indent_first=True)
if class_extra:
yield class_extra
def generate_main_class(endpoints):
yield \
"""
/**
* This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py
* Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)
*/
package water.bindings;
import water.bindings.pojos.*;
import water.bindings.proxies.retrofit.*;
import retrofit2.*;
import retrofit2.converter.gson.GsonConverterFactory;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import okhttp3.OkHttpClient;
import java.io.IOException;
import java.lang.reflect.Array;
import java.lang.reflect.Type;
import java.util.concurrent.TimeUnit;
import java.lang.reflect.Field;
public class H2oApi {
public static String DEFAULT_URL = \"http://localhost:54321/\";
public H2oApi() {
this(DEFAULT_URL);
}
public H2oApi(String url) { this.url = url; }
public H2oApi setUrl(String s) {
url = s;
retrofit = null;
return this;
}
public H2oApi setTimeout(int t) {
timeout_s = t;
retrofit = null;
return this;
}
/**
* Set time interval for job polling in {@link #waitForJobCompletion(JobKeyV3)}.
* @param millis time interval, in milliseconds
*/
public H2oApi setJobPollInterval(int millis) {
pollInterval_ms = millis;
return this;
}
/**
* Continuously poll server for the status of the given job, until it completes.
* @param jobKey job to query
* @return the finished job
*/
public JobV3 waitForJobCompletion(JobKeyV3 jobKey) {
return waitForJobCompletion(keyToString(jobKey));
}
public JobV3 waitForJobCompletion(String jobId) {
Jobs jobService = getService(Jobs.class);
Response<JobsV3> jobsResponse = null;
int retries = 3;
JobsV3 jobs = null;
do {
try {
Thread.sleep(pollInterval_ms);
jobsResponse = jobService.fetch(jobId).execute();
} catch (IOException e) {
System.err.println(\"Caught exception: \" + e);
} catch (InterruptedException e) { /* pass */ }
if (jobsResponse == null || !jobsResponse.isSuccessful())
if (retries-- > 0)
continue;
else
throw new RuntimeException(\"/3/Jobs/\" + jobId + \" failed 3 times.\");
jobs = jobsResponse.body();
if (jobs.jobs == null || jobs.jobs.length != 1)
throw new RuntimeException(\"Failed to find Job: \" + jobId);
} while (jobs != null && jobs.jobs[0].status.equals(\"RUNNING\"));
return jobs == null? null : jobs.jobs[0];
}
"""
for route in endpoints:
apiname = route["api_name"]
class_name = route["class_name"]
outtype = route["output_schema"]
input_fields = route["input_params"]
required_fields = [
field for field in input_fields if field["required"]
]
input_fields_wo_excluded = [
field for field in input_fields
if field["name"] != "_exclude_fields"
]
yield " /**"
yield bi.wrap(route["summary"], indent=" * ")
yield " */"
# Make several versions of each API call:
# (1) Only the required parameters
# (2) All parameters except the _excluded_fields
# (3) All parameters
li = len(input_fields)
le = len(input_fields_wo_excluded)
lr = len(required_fields)
assert lr <= 3, "Too many required fields in method " + apiname
if lr == li:
# The set of required fields is the same as the set of input fields. No need for (2) and (3).
input_fields = None
input_fields_wo_excluded = None
elif le == li or le == lr or li >= 4:
# If set (2) coincides with either (1) or (3), then we will not generate code for it.
# Additionally, if there are too many input params so that we will put them into a container class,
# then there will be no need for separate case (2) either.
input_fields_wo_excluded = None
# Temporary hack, since Parse reuires all the fields from ParseSetup: don't generate the required-fields-only method
if 'parse' == apiname:
required_fields = None
for fields in [
required_fields, input_fields_wo_excluded, input_fields
]:
if fields is None: continue
use_schema_param = (len(fields) >= 4)
value_field_strs = []
typed_field_strs = []
for field in fields:
ftype = translate_type(field["type"], field["schema_name"])
fname = translate_name(field["name"])
typed_field_strs.append("%s %s" % (ftype, fname))
if use_schema_param: fname = "params." + fname
if ftype.endswith("KeyV3"):
s = "keyToString(%s)" % fname
elif ftype.endswith("KeyV3[]"):
s = "keyArrayToStringArray(%s)" % fname
elif ftype.startswith("ColSpecifier"):
s = "colToString(%s)" % fname
else:
s = fname
value_field_strs.append(s)
if use_schema_param:
args = route["input_schema"] + " params"
values = "\n " + ",\n ".join(
value_field_strs) + "\n "
else:
args = ", ".join(typed_field_strs)
values = ", ".join(value_field_strs)
if fields == input_fields_wo_excluded:
values += ", \"\""
yield " public {type} {method}({args}) throws IOException {{".\
format(type=outtype, method=apiname, args=args)
yield " {clazz} s = getService({clazz}.class);".format(
clazz=class_name)
yield " return s.{method}({values}).execute().body();".\
format(method=route["handler_method"], values=values)
yield " }"
yield ""
yield \
"""
//--------- PRIVATE " + ("-" * 98)
private Retrofit retrofit;
private String url = DEFAULT_URL;
private int timeout_s = 60;
private int pollInterval_ms = 1000;
private void initializeRetrofit() {
Gson gson = new GsonBuilder()
.registerTypeAdapterFactory(new ModelV3TypeAdapter())
.registerTypeAdapter(KeyV3.class, new KeySerializer())
.registerTypeAdapter(ColSpecifierV3.class, new ColSerializer())
.registerTypeAdapter(ModelBuilderSchema.class, new ModelDeserializer())
.registerTypeAdapter(ModelSchemaBaseV3.class, new ModelSchemaDeserializer())
.create();
OkHttpClient client = new OkHttpClient.Builder()
.connectTimeout(timeout_s, TimeUnit.SECONDS)
.writeTimeout(timeout_s, TimeUnit.SECONDS)
.readTimeout(timeout_s, TimeUnit.SECONDS)
.build();
this.retrofit = new Retrofit.Builder()
.client(client)
.baseUrl(url)
.addConverterFactory(GsonConverterFactory.create(gson))
.build();
}
private Retrofit getRetrofit() {
if (retrofit == null) initializeRetrofit();
return retrofit;
}
private <T> T getService(Class<T> clazz) {
return getRetrofit().create(clazz);
}
/**
* Keys get sent as Strings and returned as objects also containing the type and URL,
* so they need a custom GSON serializer.
*/
private static class KeySerializer implements JsonSerializer<KeyV3>, JsonDeserializer<KeyV3> {
@Override
public JsonElement serialize(KeyV3 key, Type typeOfKey, JsonSerializationContext context) {
return new JsonPrimitive(key.name);
}
@Override
public KeyV3 deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) {
if (json.isJsonNull()) return null;
JsonObject jobj = json.getAsJsonObject();
String type = jobj.get(\"type\").getAsString();
switch (type) {
// TODO: dynamically generate all possible cases
case \"Key<Model>\": return context.deserialize(jobj, ModelKeyV3.class);
case \"Key<Job>\": return context.deserialize(jobj, JobKeyV3.class);
case \"Key<Grid>\": return context.deserialize(jobj, GridKeyV3.class);
case \"Key<Frame>\": return context.deserialize(jobj, FrameKeyV3.class);
default: throw new JsonParseException(\"Unable to deserialize key of type \" + type);
}
}
}
private static class ColSerializer implements JsonSerializer<ColSpecifierV3> {
@Override
public JsonElement serialize(ColSpecifierV3 col, Type typeOfCol, JsonSerializationContext context) {
return new JsonPrimitive(col.columnName);
}
}
"""
for clz, base, target_clz_factory in [
("ModelDeserializer", "ModelBuilderSchema",
lambda schema, algo: schema),
("ModelSchemaDeserializer", "ModelSchemaBaseV3",
lambda schema, algo: schema[:len(algo)] + "Model" + schema[len(algo):]
)
]:
yield " /**"
yield " * Factory method for parsing a %s json object into an instance of the model-specific subclass." % base
yield " */"
yield " private static class %s implements JsonDeserializer<%s> {" % (
clz, base)
yield " @Override"
yield " public %s deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)" % base
yield " throws JsonParseException {"
yield " if (json.isJsonNull()) return null;"
yield " if (json.isJsonObject()) {"
yield " JsonObject jobj = json.getAsJsonObject();"
yield " if (jobj.has(\"algo\")) {"
yield " String algo = jobj.get(\"algo\").getAsJsonPrimitive().getAsString().toLowerCase();"
yield " switch (algo) {"
for route in endpoints:
if route["class_name"] == "ModelBuilders" and route[
"api_name"].startswith("train"):
algo = route["algo"]
oschema = route["output_schema"]
assert oschema.lower()[:len(
algo)] == algo, "Wrong output schema for algo %s: %s" % (
algo, oschema)
model = target_clz_factory(oschema, algo)
yield " case \"{algo}\": return context.deserialize(json, {model}.class);".format(
**locals())
yield " default:"
yield " throw new JsonParseException(\"Unable to deserialize model of type \" + algo);"
yield " }"
yield " }"
yield " }"
yield " throw new JsonParseException(\"Invalid %s element \" + json.toString());" % base
yield " }"
yield " }"
yield ""
yield \
"""
def generate_main_class(endpoints):
yield dedent(0, """
/*
* This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py
* Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)
*/
package water.bindings;
import water.bindings.pojos.*;
import water.bindings.proxies.retrofit.*;
import retrofit2.*;
import retrofit2.converter.gson.GsonConverterFactory;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import okhttp3.OkHttpClient;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Type;
@SuppressWarnings("unused")
public class H2oApi {
public static String DEFAULT_URL = \"http://localhost:54321/\";
public H2oApi() {
this(DEFAULT_URL);
}
public H2oApi(String url) {
_url = url;
}
public H2oApi setUrl(String url) {
_url = url;
retrofit = null;
return this;
}
public H2oApi setTimeout(int t) {
timeout_s = t;
retrofit = null;
return this;
}
/**
* Set time interval for job polling in {@link #waitForJobCompletion(JobKeyV3)}.
* @param millis time interval, in milliseconds
*/
public H2oApi setJobPollInterval(int millis) {
pollInterval_ms = millis;
return this;
}
/**
* Continuously poll server for the status of the given job, until it completes.
* @param jobKey job to query
* @return the finished job
*/
public JobV3 waitForJobCompletion(JobKeyV3 jobKey) {
return waitForJobCompletion(keyToString(jobKey));
}
public JobV3 waitForJobCompletion(String jobId) {
Jobs jobService = getService(Jobs.class);
Response<JobsV3> jobsResponse = null;
int retries = 3;
JobsV3 jobs = null;
do {
try {
Thread.sleep(pollInterval_ms);
jobsResponse = jobService.fetch(jobId).execute();
} catch (IOException e) {
System.err.println(\"Caught exception: \" + e);
} catch (InterruptedException e) { /* pass */ }
if (jobsResponse == null || !jobsResponse.isSuccessful())
if (retries-- > 0)
continue;
else
throw new RuntimeException(\"/3/Jobs/\" + jobId + \" failed 3 times.\");
jobs = jobsResponse.body();
if (jobs.jobs == null || jobs.jobs.length != 1)
throw new RuntimeException(\"Failed to find Job: \" + jobId);
} while (jobs != null && jobs.jobs[0].status.equals(\"RUNNING\"));
return jobs == null? null : jobs.jobs[0];
}
""")
for route in endpoints:
apiname = route["api_name"]
class_name = route["class_name"]
outtype = route["output_schema"]
input_fields = route["input_params"]
required_fields = [field for field in input_fields if field["required"]]
input_fields_wo_excluded = [field for field in input_fields if field["name"] != "_exclude_fields"]
yield " /**"
yield bi.wrap(route["summary"], indent=" * ")
yield " */"
# Make several versions of each API call:
# (1) Only the required parameters
# (2) All parameters except the _excluded_fields
# (3) All parameters
li = len(input_fields)
le = len(input_fields_wo_excluded)
lr = len(required_fields)
assert lr <= 3, "Too many required fields in method " + apiname
if lr == li:
# The set of required fields is the same as the set of input fields. No need for (2) and (3).
input_fields = None
input_fields_wo_excluded = None
elif le == li or le == lr or li >= 4:
# If set (2) coincides with either (1) or (3), then we will not generate code for it.
# Additionally, if there are too many input params so that we will put them into a container class,
# then there will be no need for separate case (2) either.
input_fields_wo_excluded = None
# Temporary hack, since Parse requires all the fields from ParseSetup: don't generate the required-fields-only
# method
if 'parse' == apiname:
required_fields = None
for fields in [required_fields, input_fields_wo_excluded, input_fields]:
if fields is None: continue
use_schema_param = (len(fields) >= 4)
value_field_strs = []
typed_field_strs = []
for field in fields:
ftype = translate_type(field["type"], field["schema_name"])
fname = translate_name(field["name"])
typed_field_strs.append("%s %s" % (ftype, fname))
if use_schema_param: fname = "params." + fname
if ftype.endswith("KeyV3"):
s = "keyToString(%s)" % fname
elif ftype.endswith("KeyV3[]"):
s = "keyArrayToStringArray(%s)" % fname
elif ftype.startswith("ColSpecifier"):
s = "colToString(%s)" % fname
else:
s = fname
value_field_strs.append(s)
if use_schema_param:
args = route["input_schema"] + " params"
values = "\n " + ",\n ".join(value_field_strs) + "\n "
else:
args = ", ".join(typed_field_strs)
values = ", ".join(value_field_strs)
if fields == input_fields_wo_excluded:
values += ", \"\""
yield " public {type} {method}({args}) throws IOException {{".\
format(type=outtype, method=apiname, args=args)
yield " {clazz} s = getService({clazz}.class);".format(clazz=class_name)
yield " return s.{method}({values}).execute().body();".\
format(method=route["handler_method"], values=values)
yield " }"
yield ""
yield dedent(2, """
//--------- PRIVATE --------------------------------------------------------------------------------------------------
private Retrofit retrofit;
private String _url = DEFAULT_URL;
private int timeout_s = 60;
private int pollInterval_ms = 1000;
private void initializeRetrofit() {
Gson gson = new GsonBuilder()
.registerTypeAdapterFactory(new ModelV3TypeAdapter())
.registerTypeAdapter(KeyV3.class, new KeySerializer())
.registerTypeAdapter(ColSpecifierV3.class, new ColSerializer())
.registerTypeAdapter(ModelBuilderSchema.class, new ModelDeserializer())
.registerTypeAdapter(ModelSchemaBaseV3.class, new ModelSchemaDeserializer())
.registerTypeAdapter(ModelOutputSchemaV3.class, new ModelOutputDeserializer())
.registerTypeAdapter(ModelParametersSchemaV3.class, new ModelParametersDeserializer())
.create();
OkHttpClient client = new OkHttpClient.Builder()
.connectTimeout(timeout_s, TimeUnit.SECONDS)
.writeTimeout(timeout_s, TimeUnit.SECONDS)
.readTimeout(timeout_s, TimeUnit.SECONDS)
.build();
this.retrofit = new Retrofit.Builder()
.client(client)
.baseUrl(_url)
.addConverterFactory(GsonConverterFactory.create(gson))
.build();
}
private Retrofit getRetrofit() {
if (retrofit == null) initializeRetrofit();
return retrofit;
}
private <T> T getService(Class<T> clazz) {
return getRetrofit().create(clazz);
}
/**
* Keys get sent as Strings and returned as objects also containing the type and URL,
* so they need a custom GSON serializer.
*/
private static class KeySerializer implements JsonSerializer<KeyV3>, JsonDeserializer<KeyV3> {
@Override
public JsonElement serialize(KeyV3 key, Type typeOfKey, JsonSerializationContext context) {
return new JsonPrimitive(key.name);
}
@Override
public KeyV3 deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) {
if (json.isJsonNull()) return null;
JsonObject jobj = json.getAsJsonObject();
String type = jobj.get(\"type\").getAsString();
switch (type) {
// TODO: dynamically generate all possible cases
case \"Key<Model>\": return context.deserialize(jobj, ModelKeyV3.class);
case \"Key<Job>\": return context.deserialize(jobj, JobKeyV3.class);
case \"Key<Grid>\": return context.deserialize(jobj, GridKeyV3.class);
case \"Key<Frame>\": return context.deserialize(jobj, FrameKeyV3.class);
default: throw new JsonParseException(\"Unable to deserialize key of type \" + type);
}
}
}
private static class ColSerializer implements JsonSerializer<ColSpecifierV3> {
@Override
public JsonElement serialize(ColSpecifierV3 col, Type typeOfCol, JsonSerializationContext context) {
return new JsonPrimitive(col.columnName);
}
}
""")
def inj(middle):
return lambda schema, algo: schema[:len(algo)] + middle + schema[len(algo):]
for clz, base, target_clz_factory in [
("ModelDeserializer", "ModelBuilderSchema", lambda schema, algo: schema),
("ModelSchemaDeserializer", "ModelSchemaBaseV3", inj("Model")),
("ModelOutputDeserializer", "ModelOutputSchemaV3", inj("ModelOutput")),
("ModelParametersDeserializer", "ModelParametersSchemaV3", inj("Parameters"))]:
yield " /**"
yield " * Factory method for parsing a %s json object into an instance of the model-specific subclass." % base
yield " */"
yield " private static class %s implements JsonDeserializer<%s> {" % (clz, base)
yield " @Override"
yield " public %s deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)" % base
yield " throws JsonParseException {"
yield " if (json.isJsonNull()) return null;"
yield " if (json.isJsonObject()) {"
yield " JsonObject jobj = json.getAsJsonObject();"
yield " if (jobj.has(\"algo\")) {"
yield " String algo = jobj.get(\"algo\").getAsJsonPrimitive().getAsString().toLowerCase();"
yield " switch (algo) {"
for route in endpoints:
if route["class_name"] == "ModelBuilders" and route["api_name"].startswith("train"):
algo = route["algo"]
oschema = route["output_schema"]
assert oschema.lower()[:len(algo)] == algo, "Wrong output schema for algo %s: %s" % (algo, oschema)
model = target_clz_factory(oschema, algo)
yield " case \"{algo}\": return context.deserialize(json, {model}.class);".format(**locals())
yield " default:"
yield " throw new JsonParseException(\"Unable to deserialize model of type \" + algo);"
yield " }"
yield " }"
yield " }"
yield " throw new JsonParseException(\"Invalid %s element \" + json.toString());" % base
yield " }"
yield " }"
yield ""
yield dedent(2, """
private static class ModelV3TypeAdapter implements TypeAdapterFactory {
@Override
public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
final Class<? super T> rawType = type.getRawType();
if (!ModelBuilderSchema.class.isAssignableFrom(rawType) &&
!ModelSchemaBaseV3.class.isAssignableFrom(rawType)) return null;
final TypeAdapter<T> delegate = gson.getDelegateAdapter(this, type);
return new TypeAdapter<T>() {
@Override
public void write(JsonWriter out, T value) throws IOException {
delegate.write(out, value);
}
@Override
public T read(JsonReader in) throws IOException {
JsonObject jobj = new JsonParser().parse(in).getAsJsonObject();
if (jobj.has(\"parameters\") && jobj.get(\"parameters\").isJsonArray()) {
JsonArray jarr = jobj.get(\"parameters\").getAsJsonArray();
JsonObject paramsNew = new JsonObject();
for (JsonElement item : jarr) {
JsonObject itemObj = item.getAsJsonObject();
paramsNew.add(itemObj.get(\"name\").getAsString(), itemObj.get(\"actual_value\"));
}
jobj.add(\"parameters\", paramsNew);
}
// noinspection unchecked
return (T) new Gson().fromJson(jobj, rawType);
}
};
}
}
/**
* Return an array of Strings for an array of keys.
*/
public static String[] keyArrayToStringArray(KeyV3[] keys) {
if (keys == null) return null;
String[] ids = new String[keys.length];
int i = 0;
for (KeyV3 key : keys) ids[i++] = key.name;
return ids;
}
/**
* Return an array of keys from an array of Strings.
* @param ids array of string ids to convert to KeyV3's
* @param clz class of key objects to create. Since we have JobKeyV3, FrameKeyV3, ModelKeyV3, etc -- this
* method needs to know which of these keys you want to create
*/
public static <T extends KeyV3> T[] stringArrayToKeyArray(String[] ids, Class<T> clz) {
if (ids == null) return null;
// noinspection unchecked
T[] keys = (T[]) Array.newInstance(clz, ids.length);
String keyType = clz.getSimpleName();
if (keyType.endsWith(\"KeyV3\")) keyType = keyType.substring(0, keyType.length()-5);
try {
int i = 0;
for (String id: ids) {
keys[i] = clz.getConstructor().newInstance();
keys[i].name = id;
keys[i].type = keyType;
i++;
}
}
catch (Exception e) {
e.printStackTrace();
}
return keys;
}
/**
*
*/
public static String keyToString(KeyV3 key) {
return key == null? null : key.name;
}
/**
*
*/
public static FrameKeyV3 stringToFrameKey(String key) {
if (key == null) return null;
FrameKeyV3 k = new FrameKeyV3();
k.name = key;
return k;
}
/**
*
*/
private static String colToString(ColSpecifierV3 col) {
return col == null? null : col.columnName;
}
/**
*
*/
public static String stringToCol(String col) {
if (col == null) return null;
ColSpecifierV3 c = new ColSpecifierV3();
c.columnName = col;
return col;
}
public static void copyFields(Object to, Object from) {
Field[] fromFields = from.getClass().getDeclaredFields();
Field[] toFields = to.getClass().getDeclaredFields();
for (Field fromField : fromFields){
Field toField;
try {
toField = to.getClass().getDeclaredField(fromField.getName());
fromField.setAccessible(true);
toField.setAccessible(true);
toField.set(to, fromField.get(from));
}
catch (Exception ignored) {
// NoSuchField is the normal case
}
}
}
}
""")
def gen_module(schema, algo):
"""
Ideally we should be able to avoid logic specific to algos in this file.
Instead, customizations are externalized in ./python/gen_{algo}.py files.
Logic that is specific to python types (e.g. H2OFrame, enums as list...) should however stay here
as the type translation is done in this file.
"""
classname = algo_to_classname(algo)
extra_imports = get_customizations_for(algo, 'extensions.__imports__')
class_doc = get_customizations_for(algo, 'doc.__class__')
class_examples = get_customizations_for(algo, 'examples.__class__')
class_extras = get_customizations_for(algo, 'extensions.__class__')
module_extras = get_customizations_for(algo, 'extensions.__module__')
update_param_defaults = get_customizations_for('defaults', 'update_param')
update_param = get_customizations_for(algo, 'update_param')
deprecated_params = get_customizations_for(algo, 'deprecated_params', {})
def extend_schema_params(param):
pname = param.get('name')
param = deepcopy(param)
updates = None
for update_fn in [update_param, update_param_defaults]:
if callable(update_fn):
updates = update_fn(pname, param)
if updates is not None:
param = updates
break
# return param if isinstance(param, (list, tuple)) else [param] # always return array to support deprecated aliases
return param
extended_params = [extend_schema_params(p) for p in schema['parameters']]
param_names = []
for param in extended_params:
pname = param.get('name')
ptype = param.get('type')
pvalues = param.get('values')
pdefault = param.get('default_value')
assert (ptype[:4] == 'enum'
) == bool(pvalues), "Values are expected for enum types only"
if pvalues:
enum_values = [normalize_enum_constant(p) for p in pvalues]
if pdefault:
pdefault = normalize_enum_constant(pdefault)
else:
enum_values = None
if pname in reserved_words:
pname += "_"
param_names.append(pname)
param['pname'] = pname
param['default_value'] = pdefault
param['ptype'] = translate_type_for_check(ptype, enum_values)
param['dtype'] = translate_type_for_doc(ptype, enum_values)
if deprecated_params:
extended_params = [
p for p in extended_params
if p['pname'] not in deprecated_params.keys()
]
yield "#!/usr/bin/env python"
yield "# -*- encoding: utf-8 -*-"
yield "#"
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield "#"
yield "from __future__ import absolute_import, division, print_function, unicode_literals"
yield ""
if deprecated_params:
yield "from h2o.utils.metaclass import deprecated_params, deprecated_property"
if extra_imports:
yield reformat_block(extra_imports)
yield "from h2o.estimators.estimator_base import H2OEstimator"
yield "from h2o.exceptions import H2OValueError"
yield "from h2o.frame import H2OFrame"
yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric"
yield ""
yield ""
yield "class %s(H2OEstimator):" % classname
yield ' """'
yield " " + schema["algo_full_name"]
yield ""
if class_doc:
yield reformat_block(class_doc, 4)
if class_examples:
yield ""
yield " :examples:"
yield ""
yield reformat_block(class_examples, 4)
yield ' """'
yield ""
yield ' algo = "%s"' % algo
yield " supervised_learning = %s" % get_customizations_for(
algo, 'supervised_learning', True)
options = get_customizations_for(algo, 'options')
if options:
yield " _options_ = %s" % reformat_block(
pformat(options), prefix=' ' * 16, prefix_first=False)
yield ""
if deprecated_params:
yield reformat_block("@deprecated_params(%s)" % deprecated_params,
indent=4)
init_sig = "def __init__(self,\n%s\n):" % "\n".join(
"%s=%s, # type: %s" %
(name, default, "Optional[%s]" % type if default is None else type)
for name, default, type in [(
p.get('pname'), stringify(p.get('default_value'), infinity=None),
p.get('dtype')) for p in extended_params])
yield reformat_block(init_sig,
indent=4,
prefix=' ' * 13,
prefix_first=False)
yield ' """'
for p in extended_params:
pname, pdefault, dtype, pdoc = p.get('pname'), stringify(
p.get('default_value')), p.get('dtype'), p.get('help')
pdesc = "%s: %s\nDefaults to ``%s``." % (pname, pdoc, pdefault)
pident = ' ' * 15
yield " :param %s" % bi.wrap(
pdesc, indent=pident, indent_first=False)
yield " :type %s: %s%s" % (
pname, bi.wrap(dtype, indent=pident, indent_first=False),
", optional" if pdefault is None else "")
yield ' """'
yield " super(%s, self).__init__()" % classname
yield " self._parms = {}"
for p in extended_params:
pname = p.get('pname')
if pname == 'model_id':
yield " self._id = self._parms['model_id'] = model_id"
else:
yield " self.%s = %s" % (pname, pname)
rest_api_version = get_customizations_for(algo, 'rest_api_version')
if rest_api_version:
yield ' self._parms["_rest_version"] = %s' % rest_api_version
yield ""
for param in extended_params:
pname = param.get('pname')
if pname == "model_id":
continue # The getter is already defined in ModelBase
sname = pname[:-1] if pname[-1] == '_' else pname
ptype = param.get('ptype')
dtype = param.get('dtype')
pdefault = param.get('default_value')
if dtype.startswith("Enum"):
vals = dtype[5:-1].split(", ")
property_doc = "One of: " + ", ".join("``%s``" % v for v in vals)
else:
property_doc = "Type: ``%s``" % dtype
property_doc += ("." if pdefault is None else ", defaults to ``%s``." %
stringify(pdefault))
yield " @property"
yield " def %s(self):" % pname
yield ' """'
yield bi.wrap(param.get('help'), indent=8 *
' ') # we need to wrap only for text coming from server
yield ""
yield bi.wrap(property_doc, indent=8 * ' ')
custom_property_doc = get_customizations_for(algo,
"doc.{}".format(pname))
if custom_property_doc:
yield ""
yield reformat_block(custom_property_doc, 8)
property_examples = get_customizations_for(algo,
"examples.{}".format(pname))
if property_examples:
yield ""
yield " :examples:"
yield ""
yield reformat_block(property_examples, 8)
yield ' """'
property_getter = get_customizations_or_defaults_for(
algo, "overrides.{}.getter".format(
pname)) # check gen_stackedensemble.py for an example
if property_getter:
yield reformat_block(property_getter.format(**locals()), 8)
else:
yield " return self._parms.get(\"%s\")" % sname
yield ""
yield " @%s.setter" % pname
yield " def %s(self, %s):" % (pname, pname)
property_setter = get_customizations_or_defaults_for(
algo, "overrides.{}.setter".format(
pname)) # check gen_stackedensemble.py for an example
if property_setter:
yield reformat_block(property_setter.format(**locals()), 8)
elif "H2OFrame" in ptype:
yield " self._parms[\"%s\"] = H2OFrame._validate(%s, '%s')" % (
sname, pname, pname)
else:
yield " assert_is_type(%s, None, %s)" % (pname, ptype)
yield " self._parms[\"%s\"] = %s" % (sname, pname)
yield ""
for old, new in deprecated_params.items():
new_name = new[0] if isinstance(new, tuple) else new
yield " %s = deprecated_property('%s', %s)" % (old, old, new)
yield ""
if class_extras:
yield reformat_block(code_as_str(class_extras), 4)
if module_extras:
yield ""
yield reformat_block(code_as_str(module_extras))
def gen_module(schema, algo, module):
help_preamble = help_preamble_for(algo)
help_details = help_details_for(algo)
help_return = help_return_for(algo)
help_epilogue = help_epilogue_for(algo)
help_references = help_references_for(algo)
help_example = help_example_for(algo)
help_extra_params = help_extra_params_for(algo)
help_extra_checks = help_extra_checks_for(algo)
help_afterword = help_afterword_for(algo)
model_name = algo_to_modelname(algo)
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_R.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details) \n#'"
yield "# -------------------------- %s -------------------------- #" % model_name
if help_preamble:
lines = help_preamble.split("\n")
for line in lines:
yield "#' %s" % line.lstrip()
if help_extra_params:
lines = help_extra_params.split("\n")
for line in lines:
yield line.lstrip()
for param in schema["parameters"]:
if param["name"] in ["ignored_columns", "response_column", "max_confusion_matrix_size"]:
continue
if algo == "naivebayes":
if param["name"] == "min_sdev":
yield "#' @param threshold The minimum standard deviation to use for observations without enough data. "
yield "#' Must be at least 1e-10."
continue
if param["name"] == "eps_sdev":
yield "#' @param eps A threshold cutoff to deal with numeric instability, must be positive."
continue
if param["name"] in ["min_prob", "eps_prob"]:
continue
if param["name"] == "seed":
yield "#' @param seed Seed for random numbers (affects certain parts of the algo that are stochastic and those might or might not be enabled by default)"
if algo in ["deeplearning", "deepwater"]:
yield "#' Note: only reproducible when running single threaded."
yield "#' Defaults to -1 (time-based random number)."
continue
phelp = param["help"]
if param["type"] == "boolean":
phelp = "\code{Logical}. " + phelp
if param["values"]:
phelp += " Must be one of: %s." % ", ".join('"%s"' % p for p in param["values"])
if param["default_value"] is not None:
phelp += " Defaults to %s." % normalize_value(param, True)
yield "#' @param %s %s" % (param["name"], bi.wrap(phelp, indent=("#' "), indent_first=False))
if help_details:
yield "#' @details %s" % bi.wrap(help_details, indent=("#' "), indent_first=False)
if help_return:
lines = help_return.split("\n")
for line in lines:
yield "%s" % line.lstrip()
# yield "#' @return %s" % bi.wrap(help_return, indent=("#' "), indent_first=False)
if help_epilogue:
yield "#' @seealso %s" % bi.wrap(help_epilogue, indent=("#' "), indent_first=False)
if help_references:
yield "#' @references %s" % help_references
if help_example:
yield "#' @examples"
lines = help_example.split("\n")
for line in lines:
yield "#' %s" % line.lstrip()
yield "#' @export"
yield "h2o.%s <- function(%s," % (module, get_extra_params_for(algo))
# yield indent("training_frame,", 17 + len(algo))
list = []
for param in schema["parameters"]:
if param["name"] in ["ignored_columns", "response_column", "max_confusion_matrix_size", "training_frame"]:
continue
if algo == "naivebayes":
if param["name"] == "min_sdev":
list.append(indent("threshold = %s" % normalize_value(param), 17 + len(module)))
continue
if param["name"] == "eps_sdev":
list.append(indent("eps = %s" % normalize_value(param), 17 + len(module)))
continue
if param["name"] in ["min_prob", "eps_prob"]:
continue
list.append(indent("%s = %s" % (param["name"], normalize_value(param)), 17 + len(module)))
yield ",\n".join(list)
yield indent(") \n{", 17 + len(module))
if algo in ["deeplearning", "deepwater", "drf", "gbm", "glm", "naivebayes", "stackedensemble"]:
yield " #If x is missing, then assume user wants to use all columns as features."
yield " if(missing(x)){"
yield " if(is.numeric(y)){"
yield " x <- setdiff(col(training_frame),y)"
yield " }else{"
yield " x <- setdiff(colnames(training_frame),y)"
yield " }"
yield " }"
if algo == "gbm":
yield " # Required maps for different names params, including deprecated params"
yield " .gbm.map <- c(\"x\" = \"ignored_columns\","
yield " \"y\" = \"response_column\")"
elif algo == "naivebayes":
yield " .naivebayes.map <- c(\"x\" = \"ignored_columns\", \"y\" = \"response_column\", \n \
\"threshold\" = \"min_sdev\", \"eps\" = \"eps_sdev\")"
elif algo == "glm":
yield " # if (!is.null(beta_constraints)) {"
yield " # if (!inherits(beta_constraints, 'data.frame') && !is.H2OFrame(beta_constraints))"
yield " # stop(paste('`beta_constraints` must be an H2OH2OFrame or R data.frame. Got: ', class(beta_constraints)))"
yield " # if (inherits(beta_constraints, 'data.frame')) {"
yield " # beta_constraints <- as.h2o(beta_constraints)"
yield " # }"
yield " # }"
yield " if (inherits(beta_constraints, 'data.frame')) {"
yield " beta_constraints <- as.h2o(beta_constraints)"
yield " }"
yield ""
yield " # Required args: training_frame"
yield " if( missing(training_frame) ) stop(\"argument \'training_frame\' is missing, with no default\")"
# yield " if( missing(validation_frame) ) validation_frame = NULL"
yield " # Training_frame must be a key or an H2OFrame object"
yield " if (!is.H2OFrame(training_frame))"
yield " tryCatch(training_frame <- h2o.getFrame(training_frame),"
yield " error = function(err) {"
yield " stop(\"argument \'training_frame\' must be a valid H2OFrame or key\")"
yield " })"
if algo not in ["stackedensemble", "word2vec"]:
yield " # Validation_frame must be a key or an H2OFrame object"
yield " if (!is.null(validation_frame)) {"
yield " if (!is.H2OFrame(validation_frame))"
yield " tryCatch(validation_frame <- h2o.getFrame(validation_frame),"
yield " error = function(err) {"
yield " stop(\"argument \'validation_frame\' must be a valid H2OFrame or key\")"
yield " })"
yield " }"
yield " # Parameter list to send to model builder"
yield " parms <- list()"
yield " parms$training_frame <- training_frame"
if algo == "glrm":
yield " if(!missing(cols))"
yield " parms$ignored_columns <- .verify_datacols(training_frame, cols)$cols_ignore"
elif algo in ["deeplearning", "deepwater", "drf", "gbm", "glm", "naivebayes", "stackedensemble"]:
if any(param["name"] == "autoencoder" for param in schema["parameters"]):
yield " args <- .verify_dataxy(training_frame, x, y, autoencoder)"
else:
yield " args <- .verify_dataxy(training_frame, x, y)"
if any(param["name"] == "offset_column" for param in schema["parameters"]):
yield " if( !missing(offset_column) && !is.null(offset_column)) args$x_ignore <- args$x_ignore[!( offset_column == args$x_ignore )]"
if any(param["name"] == "weights_column" for param in schema["parameters"]):
yield " if( !missing(weights_column) && !is.null(weights_column)) args$x_ignore <- args$x_ignore[!( weights_column == args$x_ignore )]"
if algo != "stackedensemble":
yield " if( !missing(fold_column) && !is.null(fold_column)) args$x_ignore <- args$x_ignore[!( fold_column == args$x_ignore )]"
yield " parms$ignored_columns <- args$x_ignore"
yield " parms$response_column <- args$y\n"
elif algo == "word2vec":
yield ""
else:
yield " if(!missing(x))"
yield " parms$ignored_columns <- .verify_datacols(training_frame, x)$cols_ignore"
if algo == "svd":
yield " if(!missing(destination_key)) {"
yield " warning(\"'destination_key' is deprecated; please use 'model_id' instead.\")"
yield " if(missing(model_id)) {"
yield " parms$model_id <- destination_key"
yield " }"
yield " }"
if algo == "stackedensemble":
yield " if (!missing(model_id))"
yield " parms$model_id <- model_id"
for param in schema["parameters"]:
if param["name"] in ["ignored_columns", "response_column", "training_frame", "max_confusion_matrix_size"]:
continue
if algo == "glm" and param["name"] in ["interactions", "nfolds", "beta_constraints", "missing_values_handling"]:
continue
if param["name"] == "loss":
yield " if(!missing(loss)) {"
yield " if(loss == \"MeanSquare\") {"
yield " warning(\"Loss name 'MeanSquare' is deprecated; please use 'Quadratic' instead.\")"
yield " parms$loss <- \"Quadratic\""
yield " } else "
yield " parms$loss <- loss"
yield " }"
continue
if param["name"] in ["min_sdev", "min_prob"]:
yield " if (!missing(threshold))"
yield " parms$%s <- threshold" % param["name"]
continue
if param["name"] in ["eps_sdev", "eps_prob"]:
yield " if (!missing(eps))"
yield " parms$%s <- eps" % param["name"]
continue
yield " if (!missing(%s))" % param["name"]
yield " parms$%s <- %s" % (param["name"], param["name"])
if help_extra_checks:
lines = help_extra_checks.split("\n")
for line in lines:
yield "%s" % line
if algo != "glm":
yield " # Error check and build model"
yield " .h2o.modelJob('%s', parms, h2oRestApiVersion=%d) \n}" % (algo, 99 if algo in ["svd", "stackedensemble"] else 3)
if help_afterword:
lines = help_afterword.split("\n")
for line in lines:
yield line.lstrip()
def generate_main_class(endpoints):
yield "/**"
yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
yield " * Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield " */"
yield "package water.bindings;"
yield ""
yield "import water.bindings.pojos.*;"
yield "import water.bindings.proxies.retrofit.*;"
yield "import retrofit2.*;"
yield "import retrofit2.converter.gson.GsonConverterFactory;"
yield "import com.google.gson.*;"
yield "import okhttp3.OkHttpClient;"
yield "import java.io.IOException;"
yield "import java.lang.reflect.Type;"
yield "import java.util.concurrent.TimeUnit;"
yield ""
yield "public class H2oApi {"
yield ""
yield " public H2oApi() {}"
yield " public H2oApi(String url) { this.url = url; }"
yield ""
yield " public H2oApi setUrl(String s) {"
yield " url = s;"
yield " retrofit = null;"
yield " return this;"
yield " }"
yield ""
yield " public H2oApi setTimeout(int t) {"
yield " timeout_s = t;"
yield " retrofit = null;"
yield " return this;"
yield " }"
yield ""
yield " /**"
yield " * Set time interval for job polling in {@link #waitForJobCompletion(JobKeyV3)}."
yield " * @param millis time interval, in milliseconds"
yield " */"
yield " public H2oApi setJobPollInterval(int millis) {"
yield " pollInterval_ms = millis;"
yield " return this;"
yield " }"
yield ""
yield " /**"
yield " * Continuously poll server for the status of the given job, until it completes."
yield " * @param jobKey job to query"
yield " * @return the finished job"
yield " */"
yield " public JobV3 waitForJobCompletion(JobKeyV3 jobKey) {"
yield " return waitForJobCompletion(keyToString(jobKey));"
yield " }"
yield " public JobV3 waitForJobCompletion(String jobId) {"
yield " Jobs jobService = getService(Jobs.class);"
yield " Response<JobsV3> jobsResponse = null;"
yield " int retries = 3;"
yield " JobsV3 jobs = null;"
yield " do {"
yield " try {"
yield " Thread.sleep(pollInterval_ms);"
yield " jobsResponse = jobService.fetch(jobId).execute();"
yield " } catch (IOException e) {"
yield " System.err.println(\"Caught exception: \" + e);"
yield " } catch (InterruptedException e) { /* pass */ }"
yield " if (jobsResponse == null || !jobsResponse.isSuccessful())"
yield " if (retries-- > 0)"
yield " continue;"
yield " else"
yield " throw new RuntimeException(\"/3/Jobs/\" + jobId + \" failed 3 times.\");"
yield " jobs = jobsResponse.body();"
yield " if (jobs.jobs == null || jobs.jobs.length != 1)"
yield " throw new RuntimeException(\"Failed to find Job: \" + jobId);"
yield " } while (jobs != null && jobs.jobs[0].status.equals(\"RUNNING\"));"
yield " return jobs == null? null : jobs.jobs[0];"
yield " }"
yield ""
for route in endpoints:
apiname = route["api_name"]
class_name = route["class_name"]
outtype = route["output_schema"]
input_fields = route["input_params"]
required_fields = [field for field in input_fields if field["required"]]
input_fields_wo_excluded = [field for field in input_fields if field["name"] != "_exclude_fields"]
yield " /**"
yield bi.wrap(route["summary"], indent=" * ")
yield " */"
# Make several versions of each API call:
# (1) Only the required parameters
# (2) All parameters except the _excluded_fields
# (3) All parameters
li = len(input_fields)
le = len(input_fields_wo_excluded)
lr = len(required_fields)
assert lr <= 3, "Too many required fields in method " + apiName
if lr == li:
# The set of required fields is the same as the set of input fields. No need for (2) and (3).
input_fields = None
input_fields_wo_excluded = None
elif le == li or le == lr or li >= 4:
# If set (2) coincides with either (1) or (3), then we will not generate code for it.
# Additionally, if there are too many input params so that we will put them into a container class,
# then there will be no need for separate case (2) either.
input_fields_wo_excluded = None
# Temporary hack, since Parse reuires all the fields from ParseSetup: don't generate the required-fields-only method
if 'parse' == apiname:
required_fields = None
for fields in [required_fields, input_fields_wo_excluded, input_fields]:
if fields is None: continue
use_schema_param = (len(fields) >= 4)
value_field_strs = []
typed_field_strs = []
for field in fields:
ftype = translate_type(field["type"], field["schema_name"])
fname = translate_name(field["name"])
typed_field_strs.append("%s %s" % (ftype, fname))
if use_schema_param: fname = "params." + fname
if ftype.endswith("KeyV3"):
s = "keyToString(%s)" % fname
elif ftype.endswith("KeyV3[]"):
s = "keyArrayToStringArray(%s)" % fname
elif ftype.startswith("ColSpecifier"):
s = "colToString(%s)" % fname
else:
s = fname
value_field_strs.append(s)
if use_schema_param:
args = route["input_schema"] + " params"
values = "\n " + ",\n ".join(value_field_strs) + "\n "
else:
args = ", ".join(typed_field_strs)
values = ", ".join(value_field_strs)
if fields == input_fields_wo_excluded:
values += ", \"\""
yield " public {type} {method}({args}) throws IOException {{".\
format(type=outtype, method=apiname, args=args)
yield " {clazz} s = getService({clazz}.class);".format(clazz=class_name)
yield " return s.{method}({values}).execute().body();".\
format(method=route["handler_method"], values=values);
yield " }"
yield ""
yield ""
yield " //--------- PRIVATE " + "-"*98
yield ""
yield " private Retrofit retrofit;"
yield " private String url = \"http://*****:*****@Override"
yield " public JsonElement serialize(KeyV3 key, Type typeOfKey, JsonSerializationContext context) {"
yield " return new JsonPrimitive(key.name);"
yield " }"
yield " }"
yield " private static class ColSerializer implements JsonSerializer<ColSpecifierV3> {"
yield " @Override"
yield " public JsonElement serialize(ColSpecifierV3 col, Type typeOfCol, JsonSerializationContext context) {"
yield " return new JsonPrimitive(col.columnName);"
yield " }"
yield " }"
yield " /**"
yield " * Factory method for parsing a ModelsV3 json object into an instance of the model-specific subclass."
yield " */"
yield " private static class ModelDeserializer implements JsonDeserializer<ModelsV3> {"
yield " @Override"
yield " public ModelsV3 deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)"
yield " throws JsonParseException {"
yield " if (json.isJsonNull()) return null;"
yield " if (json.isJsonObject()) {"
yield " JsonObject jobj = json.getAsJsonObject();"
yield " if (jobj.has(\"algo\")) {"
yield " String algo = jobj.get(\"algo\").getAsJsonPrimitive().getAsString().toLowerCase();"
yield " switch (algo) {"
for route in endpoints:
if route["class_name"] == "ModelBuilders" and route["api_name"].startswith("train"):
algo = route["algo"]
oschema = route["output_schema"]
assert oschema.lower()[:len(algo)] == algo, "Wrong output schema for algo %s: %s" % (algo, oschema)
model = oschema[:len(algo)] + "Model" + oschema[len(algo):] # "DeepLearningV3" => "DeepLearningModelV3"
yield " case \"{algo}\": return context.deserialize(json, {model}.class);".format(**locals())
yield " default:"
yield " throw new JsonParseException(\"Unable to deserialize model of type \" + algo);"
yield " }"
yield " }"
yield " }"
yield " throw new JsonParseException(\"Invalid ModelsV3 element \" + json.toString());"
yield " }"
yield " }"
yield ""
yield " /**"
yield " * Return an array of Strings for an array of keys."
yield " */"
yield " public static String[] keyArrayToStringArray(KeyV3[] keys) {"
yield " if (keys == null) return null;"
yield " String[] ids = new String[keys.length];"
yield " int i = 0;"
yield " for (KeyV3 key : keys) ids[i++] = key.name;"
yield " return ids;"
yield " }"
yield ""
yield " /**"
yield " * Return an array of Keys for an array of Strings."
yield " */"
yield " public static FrameKeyV3[] stringArrayToFrameKeyArray(String[] keys) {"
yield " if (keys == null) return null;"
yield " FrameKeyV3[] ids = new FrameKeyV3[keys.length];"
yield " int i = 0;"
yield " for (String key : keys) { FrameKeyV3 k = new FrameKeyV3(); k.name = key; ids[i++] = k; }"
yield " return ids;"
yield " }"
yield ""
yield " /**"
yield " *"
yield " */"
yield " public static String keyToString(KeyV3 key) {"
yield " return key == null? null : key.name;"
yield " }"
yield ""
yield " /**"
yield " *"
yield " */"
yield " public static FrameKeyV3 stringToFrameKey(String key) {"
yield " if (key == null) return null;"
yield " FrameKeyV3 k = new FrameKeyV3();"
yield " k.name = key;"
yield " return k;"
yield " }"
yield ""
yield " /**"
yield " *"
yield " */"
yield " private static String colToString(ColSpecifierV3 col) {"
yield " return col == null? null : col.columnName;"
yield " }"
yield ""
yield " /**"
yield " *"
yield " */"
yield " public static String stringToCol(String col) {"
yield " if (col == null) return null;"
yield " ColSpecifierV3 c = new ColSpecifierV3();"
yield " c.columnName = col;"
yield " return col;"
yield " }"
yield ""
yield "}"
def generate_proxy(classname, endpoints):
"""
Retrofit interfaces look like this:
public interface GitHubService {
@GET("/users/{user}/repos")
Call<List<Repo>> listRepos(@Path("user") String user);
}
:param classname: name of the class
:param endpoints: list of endpoints served by this class
"""
# Replace path vars like (?<schemaname>.*) with {schemaname} for Retrofit's annotation
var_pattern = re.compile(r"\{(\w+)\}")
helper_class = []
found_key_array_parameter = False
yield "/**"
yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
yield " * Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield " */"
yield "package water.bindings.proxies.retrofit;"
yield ""
yield "import water.bindings.pojos.*;"
yield "import retrofit2.*;"
yield "import retrofit2.http.*;"
yield ""
yield "public interface " + classname + " {"
yield ""
for e in endpoints:
method = e["handler_method"]
param_strs = []
required_param_strs = []
for field in e["input_params"]:
fname = field["name"]
ftype = "Path" if field["is_path_param"] else "Field"
ptype = translate_type(field["type"], field["schema_name"])
if ptype.endswith("KeyV3") or ptype == "ColSpecifierV3": ptype = "String"
if ptype.endswith("KeyV3[]"): ptype = "String[]"
param_str = "@{ftype}(\"{fname}\") {ptype} {fname}".format(**locals())
param_strs.append(param_str)
if field["required"]:
required_param_strs.append(param_str)
if len(param_strs) == len(required_param_strs): required_param_strs = None
yield u" /** "
yield bi.wrap(e["summary"], indent=" * ")
for field in e["input_params"]:
s = " * @param %s " % field["name"]
yield s + bi.wrap(field["help"], indent=" *"+" "*(len(s)-4), indent_first=False)
yield u" */"
# Create 2 versions of each call: first with all input parameters present, and then only with required params
for params in [param_strs, required_param_strs]:
if params is None: continue
yield u" @FormUrlEncoded" if e["http_method"] == "POST" else None
yield u" @{method}(\"{path}\")".format(method=e["http_method"], path=e["url_pattern"])
if len(params) <= 1:
args = params[0] if params else ""
yield " Call<{schema}> {method}({args});".format(schema=e["output_schema"], method=method, args=args)
else:
yield " Call<{schema}> {method}(".format(schema=e["output_schema"], method=method)
for arg in params:
yield " " + arg + ("" if arg == params[-1] else ",")
yield " );"
yield ""
# Make special static Helper class for Grid and ModelBuilders.
if "algo" in e:
# We make two train_ and validate_ methods. One (built here) takes the parameters schema, the other
# (built above) takes each parameter.
helper_class.append(" /**")
helper_class.append(bi.wrap(e["summary"], indent=" * "))
helper_class.append(" */")
helper_class.append(" public static Call<{oschema}> {method}({outer_class} z, {ischema} p) {{"
.format(ischema=e["input_schema"], oschema=e["output_schema"], method=method,
outer_class=classname))
helper_class.append(" return z.{method}(".format(method=method))
for field in e["input_params"]:
ptype = translate_type(field["type"], field["schema_name"])
pname = translate_name(field["name"])
if ptype.endswith("KeyV3"):
s = "(p.{parm} == null? null : p.{parm}.name)".format(parm=pname)
elif ptype.endswith("KeyV3[]"):
found_key_array_parameter = True
s = "(p.{parm} == null? null : keyArrayToStringArray(p.{parm}))".format(parm=pname)
elif ptype.startswith("ColSpecifier"):
s = "(p.{parm} == null? null : p.{parm}.columnName)".format(parm=pname)
else:
s = "p." + pname
if field != e["input_params"][-1]:
s += ","
helper_class.append(" " + s)
helper_class.append(" );")
helper_class.append(" }")
helper_class.append("")
if helper_class:
yield ""
yield " class Helper {"
for line in helper_class:
yield line
if found_key_array_parameter:
yield " /**"
yield " * Return an array of Strings for an array of keys."
yield " */"
yield " public static String[] keyArrayToStringArray(KeyV3[] keys) {"
yield " if (keys == null) return null;"
yield " String[] ids = new String[keys.length];"
yield " int i = 0;"
yield " for (KeyV3 key : keys) ids[i++] = key.name;"
yield " return ids;"
yield " }"
yield " }"
yield ""
yield "}"
def generate_proxy(classname, endpoints):
"""
Generate a Retrofit Proxy class.
Retrofit interfaces look like this:
public interface GitHubService {
@GET("/users/{user}/repos")
Call<List<Repo>> listRepos(@Path("user") String user);
}
:param classname: name of the class
:param endpoints: list of endpoints served by this class
"""
# Replace path vars like (?<schemaname>.*) with {schemaname} for Retrofit's annotation
var_pattern = re.compile(r"\{(\w+)\}")
helper_class = []
found_key_array_parameter = False
yield "/*"
yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
yield " * Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield " */"
yield "package water.bindings.proxies.retrofit;"
yield ""
yield "import water.bindings.pojos.*;"
yield "import retrofit2.*;"
yield "import retrofit2.http.*;"
yield "import java.util.Map;" if classname == "Grid" or classname == "ModelBuilders" else None
yield ""
yield "public interface " + classname + " {"
yield ""
for e in endpoints:
method = e["handler_method"]
# should we always use e.api_name ?
if method == "exec":
method = e["api_name"]
param_strs = []
required_param_strs = []
for field in e["input_params"]:
fname = field["name"]
if field["is_path_param"]:
ftype = "Path"
else:
if e["http_method"] == "GET":
ftype = "Query"
else:
ftype = "Field"
ptype = translate_type(field["type"], field["schema_name"])
if ptype.endswith("KeyV3") or ptype == "ColSpecifierV3":
ptype = "String"
if ptype.endswith("KeyV3[]"): ptype = "String[]"
param_str = "@{ftype}(\"{fname}\") {ptype} {fname}".format(
**locals())
param_strs.append(param_str)
if field["required"]:
required_param_strs.append(param_str)
if len(param_strs) == len(required_param_strs):
required_param_strs = None
yield u" /** "
yield bi.wrap(e["summary"], indent=" * ")
for field in e["input_params"]:
s = " * @param %s " % field["name"]
yield s + bi.wrap(field["help"],
indent=" *" + " " * (len(s) - 4),
indent_first=False)
yield u" */"
# Create 2 versions of each call: first with all input parameters present, and then only with required params
for params in [param_strs, required_param_strs]:
if params is None: continue
yield u" @FormUrlEncoded" if e["http_method"] == "POST" else None
yield u" @{method}(\"{path}\")".format(method=e["http_method"],
path=e["url_pattern"])
if len(params) <= 1:
args = params[0] if params else ""
yield " Call<{schema}> {method}({args});".format(
schema=e["output_schema"], method=method, args=args)
else:
yield " Call<{schema}> {method}(".format(
schema=e["output_schema"], method=method)
for arg in params:
yield " " + arg + ("" if arg == params[-1] else ",")
yield " );"
yield ""
# Make special static Helper class for Grid and ModelBuilders.
if "algo" in e:
# We make two train_ and validate_ methods. One (built here) takes the parameters schema, the other
# (built above) takes each parameter.
helper_class.append(" /**")
helper_class.append(bi.wrap(e["summary"], indent=" * "))
helper_class.append(" */")
helper_class.append(
" public static Call<{oschema}> {method}({outer_class} z, {ischema} p) {{"
.format(ischema=e["input_schema"],
oschema=e["output_schema"],
method=method,
outer_class=classname))
helper_class.append(
" return z.{method}(".format(method=method))
for field in e["input_params"]:
ptype = translate_type(field["type"], field["schema_name"])
pname = translate_name(field["name"])
if ptype.endswith("KeyV3"):
s = "(p.{parm} == null? null : p.{parm}.name)".format(
parm=pname)
elif ptype.endswith("KeyV3[]"):
found_key_array_parameter = True
s = "(p.{parm} == null? null : keyArrayToStringArray(p.{parm}))".format(
parm=pname)
elif ptype == "ColSpecifierV3":
s = "(p.{parm} == null? null : p.{parm}.columnName)".format(
parm=pname)
else:
s = "p." + pname
if field != e["input_params"][-1]:
s += ","
helper_class.append(" " + s)
helper_class.append(" );")
helper_class.append(" }")
helper_class.append("")
if helper_class:
yield ""
yield " @SuppressWarnings(\"unused\")"
yield " class Helper {"
for line in helper_class:
yield line
if found_key_array_parameter:
yield " /**"
yield " * Return an array of Strings for an array of keys."
yield " */"
yield " public static String[] keyArrayToStringArray(KeyV3[] keys) {"
yield " if (keys == null) return null;"
yield " String[] ids = new String[keys.length];"
yield " int i = 0;"
yield " for (KeyV3 key : keys) ids[i++] = key.name;"
yield " return ids;"
yield " }"
yield " }"
yield ""
yield "}"
def generate_schema(class_name, schema):
"""
Generate schema POJO file.
:param class_name: name of the class
:param schema: information about the class
"""
has_map = False
is_model_builder = False
has_inherited = False
for field in schema["fields"]:
if field["name"] == "__meta": continue
if field["is_inherited"]:
has_inherited = True
continue
if field["type"].startswith("Map"): has_map = True
if field["name"] == "can_build": is_model_builder = True
superclass = schema["superclass"]
if superclass == "Schema": superclass = "Object"
fields = []
for field in schema["fields"]:
if field["name"] == "__meta": continue
java_type = translate_type(field["type"], field["schema_name"])
java_value = get_java_value(field)
# hackery: we flatten the parameters up into the ModelBuilder schema, rather than nesting them in the
# parameters schema class...
if False and is_model_builder and field["name"] == "parameters":
fields.append(("parameters", "null", "ModelParameterSchemaV3[]", field["help"], field["is_inherited"]))
else:
fields.append((field["name"], java_value, java_type, field["help"], field["is_inherited"]))
yield "/**"
yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
yield " * Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details)"
yield " */"
yield "package water.bindings.pojos;"
yield ""
yield "import com.google.gson.Gson;"
yield "import com.google.gson.annotations.*;"
yield "import java.util.Map;" if has_map else None
yield ""
yield ""
yield "public class %s extends %s {" % (class_name, superclass) if superclass != "Object" else None
yield "public class %s {" % (class_name) if superclass == "Object" else None
yield ""
for name, value, ftype, fhelp, inherited in fields:
if inherited: continue
ccname = translate_name(name)
yield " /**"
yield bi.wrap(fhelp, indent=" * ")
yield " */"
yield " @SerializedName(\"%s\")" % name if name != ccname else None
yield " public %s %s;" % (ftype, ccname)
yield ""
if has_inherited:
yield ""
yield " /*" + ("-" * 114)
yield " //" + (" " * 50) + "INHERITED"
yield " //" + ("-" * 114)
yield ""
for name, value, ftype, fhelp, inherited in fields:
if not inherited: continue
yield bi.wrap(fhelp, " // ")
yield " public %s %s;" % (ftype, translate_name(name))
yield ""
yield " */"
yield ""
yield " /**"
yield " * Public constructor"
yield " */"
yield " public %s() {" % class_name
for name, value, _, _, _ in fields:
if name == "parameters": continue
if value == "null": continue
yield " %s = %s;" % (translate_name(name), value)
yield " }"
yield ""
yield " /**"
yield " * Return the contents of this object as a JSON String."
yield " */"
yield " @Override"
yield " public String toString() {"
yield " return new Gson().toJson(this);"
yield " }"
yield ""
yield "}"
def gen_module(schema, algo, module):
help_preamble = help_preamble_for(algo)
help_details = help_details_for(algo)
help_return = help_return_for(algo)
help_epilogue = help_epilogue_for(algo)
help_references = help_references_for(algo)
help_example = help_example_for(algo)
help_extra_params = help_extra_params_for(algo)
help_extra_checks = help_extra_checks_for(algo)
help_afterword = help_afterword_for(algo)
model_name = algo_to_modelname(algo)
yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_R.py"
yield "# Copyright 2016 H2O.ai; Apache License Version 2.0 (see LICENSE for details) \n#'"
yield "# -------------------------- %s -------------------------- #" % model_name
if help_preamble:
lines = help_preamble.split("\n")
for line in lines:
yield "#' %s" % line.lstrip()
if help_extra_params:
lines = help_extra_params.split("\n")
for line in lines:
yield line.lstrip()
for param in schema["parameters"]:
if param["name"] in ["ignored_columns", "response_column", "max_confusion_matrix_size"]:
continue
if algo == "naivebayes":
if param["name"] == "min_sdev":
yield "#' @param threshold The minimum standard deviation to use for observations without enough data. "
yield "#' Must be at least 1e-10."
continue
if param["name"] == "eps_sdev":
yield "#' @param eps A threshold cutoff to deal with numeric instability, must be positive."
continue
if param["name"] in ["min_prob", "eps_prob"]:
continue
if param["name"] == "seed":
yield "#' @param seed Seed for random numbers (affects certain parts of the algo that are stochastic and those might or might not be enabled by default)"
if algo in ["deeplearning", "deepwater"]:
yield "#' Note: only reproducible when running single threaded."
yield "#' Defaults to -1 (time-based random number)."
continue
phelp = param["help"]
if param["type"] == "boolean":
phelp = "\code{Logical}. " + phelp
if param["values"]:
phelp += " Must be one of: %s." % ", ".join('"%s"' % p for p in param["values"])
if param["default_value"] is not None:
phelp += " Defaults to %s." % normalize_value(param, True)
yield "#' @param %s %s" % (param["name"], bi.wrap(phelp, indent=("#' "), indent_first=False))
if help_details:
yield "#' @details %s" % bi.wrap(help_details, indent=("#' "), indent_first=False)
if help_return:
lines = help_return.split("\n")
for line in lines:
yield "%s" % line.lstrip()
# yield "#' @return %s" % bi.wrap(help_return, indent=("#' "), indent_first=False)
if help_epilogue:
yield "#' @seealso %s" % bi.wrap(help_epilogue, indent=("#' "), indent_first=False)
if help_references:
yield "#' @references %s" % help_references
if help_example:
yield "#' @examples"
lines = help_example.split("\n")
for line in lines:
yield "#' %s" % line.lstrip()
yield "#' @export"
yield "h2o.%s <- function(%s," % (module, get_extra_params_for(algo))
# yield indent("training_frame,", 17 + len(algo))
list = []
for param in schema["parameters"]:
if param["name"] in ["ignored_columns", "response_column", "max_confusion_matrix_size", "training_frame"]:
continue
if algo == "naivebayes":
if param["name"] == "min_sdev":
list.append(indent("threshold = %s" % normalize_value(param), 17 + len(module)))
continue
if param["name"] == "eps_sdev":
list.append(indent("eps = %s" % normalize_value(param), 17 + len(module)))
continue
if param["name"] in ["min_prob", "eps_prob"]:
continue
list.append(indent("%s = %s" % (param["name"], normalize_value(param)), 17 + len(module)))
yield ",\n".join(list)
yield indent(") \n{", 17 + len(module))
if algo in ["deeplearning", "deepwater", "drf", "gbm", "glm", "naivebayes", "stackedensemble"]:
yield " #If x is missing, then assume user wants to use all columns as features."
yield " if(missing(x)){"
yield " if(is.numeric(y)){"
yield " x <- setdiff(col(training_frame),y)"
yield " }else{"
yield " x <- setdiff(colnames(training_frame),y)"
yield " }"
yield " }"
if algo == "gbm":
yield " # Required maps for different names params, including deprecated params"
yield " .gbm.map <- c(\"x\" = \"ignored_columns\","
yield " \"y\" = \"response_column\")"
elif algo == "naivebayes":
yield " .naivebayes.map <- c(\"x\" = \"ignored_columns\", \"y\" = \"response_column\", \n \
\"threshold\" = \"min_sdev\", \"eps\" = \"eps_sdev\")"
elif algo == "glm":
yield " # if (!is.null(beta_constraints)) {"
yield " # if (!inherits(beta_constraints, 'data.frame') && !is.H2OFrame(beta_constraints))"
yield " # stop(paste('`beta_constraints` must be an H2OH2OFrame or R data.frame. Got: ', class(beta_constraints)))"
yield " # if (inherits(beta_constraints, 'data.frame')) {"
yield " # beta_constraints <- as.h2o(beta_constraints)"
yield " # }"
yield " # }"
yield " if (inherits(beta_constraints, 'data.frame')) {"
yield " beta_constraints <- as.h2o(beta_constraints)"
yield " }"
yield ""
yield " # Required args: training_frame"
yield " if( missing(training_frame) ) stop(\"argument \'training_frame\' is missing, with no default\")"
# yield " if( missing(validation_frame) ) validation_frame = NULL"
yield " # Training_frame must be a key or an H2OFrame object"
yield " if (!is.H2OFrame(training_frame))"
yield " tryCatch(training_frame <- h2o.getFrame(training_frame),"
yield " error = function(err) {"
yield " stop(\"argument \'training_frame\' must be a valid H2OFrame or key\")"
yield " })"
if algo not in ["stackedensemble", "word2vec"]:
yield " # Validation_frame must be a key or an H2OFrame object"
yield " if (!is.null(validation_frame)) {"
yield " if (!is.H2OFrame(validation_frame))"
yield " tryCatch(validation_frame <- h2o.getFrame(validation_frame),"
yield " error = function(err) {"
yield " stop(\"argument \'validation_frame\' must be a valid H2OFrame or key\")"
yield " })"
yield " }"
yield " # Parameter list to send to model builder"
yield " parms <- list()"
yield " parms$training_frame <- training_frame"
if algo == "glrm":
yield " if(!missing(cols))"
yield " parms$ignored_columns <- .verify_datacols(training_frame, cols)$cols_ignore"
elif algo in ["deeplearning", "deepwater", "drf", "gbm", "glm", "naivebayes", "stackedensemble"]:
if any(param["name"] == "autoencoder" for param in schema["parameters"]):
yield " args <- .verify_dataxy(training_frame, x, y, autoencoder)"
else:
yield " args <- .verify_dataxy(training_frame, x, y)"
if any(param["name"] == "offset_column" for param in schema["parameters"]):
yield " if( !missing(offset_column) && !is.null(offset_column)) args$x_ignore <- args$x_ignore[!( offset_column == args$x_ignore )]"
if any(param["name"] == "weights_column" for param in schema["parameters"]):
yield " if( !missing(weights_column) && !is.null(weights_column)) args$x_ignore <- args$x_ignore[!( weights_column == args$x_ignore )]"
if algo != "stackedensemble":
yield " if( !missing(fold_column) && !is.null(fold_column)) args$x_ignore <- args$x_ignore[!( fold_column == args$x_ignore )]"
yield " parms$ignored_columns <- args$x_ignore"
yield " parms$response_column <- args$y\n"
elif algo == "word2vec":
yield ""
else:
yield " if(!missing(x))"
yield " parms$ignored_columns <- .verify_datacols(training_frame, x)$cols_ignore"
if algo == "svd":
yield " if(!missing(destination_key)) {"
yield " warning(\"'destination_key' is deprecated; please use 'model_id' instead.\")"
yield " if(missing(model_id)) {"
yield " parms$model_id <- destination_key"
yield " }"
yield " }"
if algo == "stackedensemble":
yield " if (!missing(model_id))"
yield " parms$model_id <- model_id"
for param in schema["parameters"]:
if param["name"] in ["ignored_columns", "response_column", "training_frame", "max_confusion_matrix_size"]:
continue
if algo == "glm" and param["name"] in ["interactions", "nfolds", "beta_constraints", "missing_values_handling"]:
continue
if param["name"] == "loss":
yield " if(!missing(loss)) {"
yield " if(loss == \"MeanSquare\") {"
yield " warning(\"Loss name 'MeanSquare' is deprecated; please use 'Quadratic' instead.\")"
yield " parms$loss <- \"Quadratic\""
yield " } else "
yield " parms$loss <- loss"
yield " }"
continue
if param["name"] in ["min_sdev", "min_prob"]:
yield " if (!missing(threshold))"
yield " parms$%s <- threshold" % param["name"]
continue
if param["name"] in ["eps_sdev", "eps_prob"]:
yield " if (!missing(eps))"
yield " parms$%s <- eps" % param["name"]
continue
yield " if (!missing(%s))" % param["name"]
yield " parms$%s <- %s" % (param["name"], param["name"])
if help_extra_checks:
lines = help_extra_checks.split("\n")
for line in lines:
yield "%s" % line
if algo != "glm":
yield " # Error check and build model"
yield " .h2o.modelJob('%s', parms, h2oRestApiVersion=%d) \n}" % (algo, 99 if algo in ["svd", "stackedensemble"] else 3)
if help_afterword:
lines = help_afterword.split("\n")
for line in lines:
yield line.lstrip()
