def test_python_command_execution(self):
"""Test command line execution."""
Shell.call('rm -rf tmp')
Shell.call('mkdir tmp')
filename = '{}.java'.format(self.tmp_fn)
cp_src = os.path.join('tmp', filename)
with open(cp_src, 'w') as f:
porter = Porter(self.estimator)
out = porter.export(method_name='predict', class_name=self.tmp_fn)
f.write(out)
# $ javac tmp/Tmp.java
cmd = ' '.join(['javac', cp_src])
Shell.call(cmd)
# Rename estimator for comparison:
filename = '{}_2.java'.format(self.tmp_fn)
cp_dest = os.path.join('tmp', filename)
# $ mv tmp/Brain.java tmp/Brain_2.java
cmd = ' '.join(['mv', cp_src, cp_dest])
Shell.call(cmd)
# Dump estimator:
filename = '{}.pkl'.format(self.tmp_fn)
pkl_path = os.path.join('tmp', filename)
joblib.dump(self.estimator, pkl_path)
# Port estimator:
cmd = 'python -m sklearn_porter.cli.__main__ {}' \
' --class_name Brain'.format(pkl_path)
Shell.call(cmd)
# Compare file contents:
equal = filecmp.cmp(cp_src, cp_dest)
self.assertEqual(equal, True)
def test_python_command_execution(self):
"""Test command line execution."""
Shell.call('rm -rf tmp')
Shell.call('mkdir tmp')
filename = '{}.java'.format(self.tmp_fn)
cp_src = os.path.join('tmp', filename)
with open(cp_src, 'w') as f:
porter = Porter(self.estimator)
out = porter.export(method_name='predict', class_name=self.tmp_fn)
f.write(out)
# $ javac tmp/Tmp.java
cmd = ' '.join(['javac', cp_src])
Shell.call(cmd)
# Rename estimator for comparison:
filename = '{}_2.java'.format(self.tmp_fn)
cp_dest = os.path.join('tmp', filename)
# $ mv tmp/Brain.java tmp/Brain_2.java
cmd = ' '.join(['mv', cp_src, cp_dest])
Shell.call(cmd)
# Dump estimator:
filename = '{}.pkl'.format(self.tmp_fn)
pkl_path = os.path.join('tmp', filename)
joblib.dump(self.estimator, pkl_path)
# Port estimator:
cmd = 'python -m sklearn_porter.cli.__main__ -i {}' \
' --class_name Brain'.format(pkl_path)
Shell.call(cmd)
# Compare file contents:
equal = filecmp.cmp(cp_src, cp_dest)
self.assertEqual(equal, True)
def _port_estimator(self):
self.estimator.fit(self.X, self.y)
Shell.call('rm -rf tmp')
Shell.call('mkdir tmp')
filename = self.tmp_fn + '.rb'
path = os.path.join('tmp', filename)
with open(path, 'w') as f:
porter = Porter(self.estimator, language=self.LANGUAGE)
out = porter.export(class_name='Brain', method_name='foo')
f.write(out)
def _port_estimator(self, export_data=False, embed_data=False):
self.estimator.fit(self.X, self.y)
Shell.call('rm -rf tmp')
Shell.call('mkdir tmp')
with open(self.tmp_fn, 'w') as f:
porter = Porter(self.estimator, language=self.LANGUAGE)
if export_data:
out = porter.export(class_name='Brain',
method_name='foo',
export_data=True,
export_dir='tmp')
else:
out = porter.export(class_name='Brain',
method_name='foo',
embed_data=embed_data)
f.write(out)
def pred_in_custom(self, features, cast=True):
# $ ./<temp_filename> <features>
cmd = [os.path.join('.', 'tmp', self.tmp_fn)]
args = [str(f).strip() for f in features]
cmd += args
pred = Shell.check_output(cmd)
return int(pred) if cast else float(pred)
def _test_dependencies(self):
"""
Check all target programming and operating
system dependencies.
"""
lang = self.target_language
if sys.platform in ('cygwin', 'win32', 'win64'):
error = "The required dependencies aren't available on Windows."
raise EnvironmentError(error)
# Dependencies:
depends = {
'c': ['gcc'],
'java': ['java', 'javac'],
'js': ['node'],
'go': ['go'],
'php': ['php'],
'ruby': ['ruby']
}
all_depends = depends.get(lang) + ['mkdir', 'rm']
all_depends = [str(e) for e in all_depends]
cmd = 'if hash {} 2/dev/null; then echo 1; else echo 0; fi'
for exe in all_depends:
cmd = cmd.format(exe)
status = Shell.check_output(cmd)
if sys.version_info >= (3, 3) and isinstance(status, bytes):
status = status.decode('utf-8')
status = str(status).strip()
if status != '1':
error = "The required application '{0}'" \
" isn't available.".format(exe)
raise SystemError(error)
def pred_in_custom(self, features, cast=True, export_data=False):
cmd = ['node', self.tmp_fn]
if export_data:
cmd += ['http://0.0.0.0:8713/tmp/data.json']
args = [str(f).strip() for f in features]
cmd += args
pred = Shell.check_output(cmd)
return int(pred) if cast else float(pred)
def pred_in_custom(self, features, cast=True):
# $ php -f <tmp_filename> <features>
filename = self.tmp_fn + '.php'
path = os.path.join('tmp', filename)
cmd = 'php -f {}'.format(path).split()
args = [str(f).strip() for f in features]
cmd += args
pred = Shell.check_output(cmd)
return int(pred) if cast else float(pred)
def pred_in_custom(self, features, cast=True):
# $ ruby temp <temp_filename> <features>
filename = self.tmp_fn + '.rb'
path = os.path.join('tmp', filename)
cmd = ['ruby', path]
args = [str(f).strip() for f in features]
cmd += args
pred = Shell.check_output(cmd)
return int(pred) if cast else float(pred)
def pred_in_custom(self, features, cast=True, export_data=False):
if export_data:
cmd = 'java -cp ./gson.jar:./tmp {}'.format(self.tmp_fn).split()
cmd += ['./tmp/data.json']
else:
cmd = 'java -classpath tmp {}'.format(self.tmp_fn).split()
args = [str(f).strip() for f in features]
cmd += args
pred = Shell.check_output(cmd)
return int(pred) if cast else float(pred)
def _port_estimator(self):
self.estimator.fit(self.X, self.y)
Shell.call('rm -rf tmp')
Shell.call('mkdir tmp')
path = os.path.join('.', 'tmp', self.tmp_fn + '.go')
output = os.path.join('.', 'tmp', self.tmp_fn)
with open(path, 'w') as f:
porter = Porter(self.estimator, language=self.LANGUAGE)
out = porter.export(class_name='Brain', method_name='foo')
f.write(out)
cmd = 'go build -o {} {}'.format(output, path)
Shell.call(cmd)
def _port_estimator(self):
self.estimator.fit(self.X, self.y)
Shell.call('rm -rf tmp')
Shell.call('mkdir tmp')
filename = self.tmp_fn + '.c'
path = os.path.join('tmp', filename)
with open(path, 'w') as f:
porter = Porter(self.estimator, language=self.LANGUAGE)
out = porter.export(class_name='Brain', method_name='foo')
f.write(out)
# $ gcc temp/tmp.c -o temp/tmp
cmd = 'gcc {} -std=c99 -lm -o tmp/{}'.format(path, self.tmp_fn)
Shell.call(cmd)
def _port_estimator(self, export_data=False, embed_data=False):
self.estimator.fit(self.X, self.y)
Shell.call('rm -rf tmp')
Shell.call('mkdir tmp')
filename = self.tmp_fn + '.java'
path = os.path.join('tmp', filename)
with open(path, 'w') as f:
porter = Porter(self.estimator, language=self.LANGUAGE)
if export_data:
out = porter.export(class_name='Brain',
method_name='foo',
export_data=True,
export_dir='tmp')
else:
out = porter.export(class_name='Brain',
method_name='foo',
embed_data=embed_data)
f.write(out)
if export_data:
cmd = 'javac -cp ./gson.jar {}'.format(path)
Shell.call(cmd)
else:
cmd = 'javac ' + path
Shell.call(cmd)
def test_check_output_echo_list_xyz(self):
self.assertEqual(Shell.check_output(['echo', 'xyz'], shell=False), 'xyz')
def test_check_output_echo_num_1(self):
self.assertEqual(Shell.check_output('echo 1'), '1')
def test_check_output_echo_list_xyz(self):
self.assertEqual(Shell.check_output(['echo', 'xyz'], shell=False),
'xyz')
def predict(self,
X,
class_name=None,
method_name=None,
tnp_dir='tmp',
keep_tmp_dir=False,
num_format=lambda x: str(x)):
"""
Predict using the transpiled model.
Parameters
----------
:param X : {array-like}, shape (n_features) or (n_samples, n_features)
The input data.
:param class_name : string, default: None
The name for the ported class.
:param method_name : string, default: None
The name for the ported method.
:param tnp_dir : string, default: 'tmp'
The path to the temporary directory for
storing the transpiled (and compiled) model.
:param keep_tmp_dir : bool, default: False
Whether to delete the temporary directory
or not.
:param num_format : lambda x, default: lambda x: str(x)
The representation of the floating-point values.
Returns
-------
y : int or array-like, shape (n_samples,)
The predicted class or classes.
"""
if class_name is None:
class_name = self.estimator_name
if method_name is None:
method_name = self.target_method
# Dependencies:
if not self._tested_dependencies:
self._test_dependencies()
self._tested_dependencies = True
# Support:
if 'predict' not in set(self.template.SUPPORTED_METHODS):
error = "Currently the given model method" \
" '{}' isn't supported.".format('predict')
raise AttributeError(error)
# Cleanup:
Shell.call('rm -rf {}'.format(tnp_dir))
Shell.call('mkdir {}'.format(tnp_dir))
# Transpiled model:
details = self.export(class_name=class_name,
method_name=method_name,
num_format=num_format,
details=True)
filename = Porter._get_filename(class_name, self.target_language)
target_file = os.path.join(tnp_dir, filename)
with open(target_file, str('w')) as file_:
file_.write(details.get('estimator'))
# Compilation command:
comp_cmd = details.get('cmd').get('compilation')
if comp_cmd is not None:
Shell.call(comp_cmd, cwd=tnp_dir)
# Execution command:
exec_cmd = details.get('cmd').get('execution')
exec_cmd = str(exec_cmd).split()
pred_y = None
# Single feature set:
if exec_cmd is not None and len(X.shape) == 1:
full_exec_cmd = exec_cmd + [str(sample).strip() for sample in X]
pred_y = Shell.check_output(full_exec_cmd, cwd=tnp_dir)
pred_y = int(pred_y)
# Multiple feature sets:
if exec_cmd is not None and len(X.shape) > 1:
pred_y = np.empty(X.shape[0], dtype=int)
for idx, features in enumerate(X):
full_exec_cmd = exec_cmd + [str(f).strip() for f in features]
pred = Shell.check_output(full_exec_cmd, cwd=tnp_dir)
pred_y[idx] = int(pred)
# Cleanup:
if not keep_tmp_dir:
Shell.call('rm -rf {}'.format(tnp_dir))
return pred_y
def test_check_output_echo_num_0(self):
self.assertEqual(Shell.check_output('echo 0'), '0')
def test_check_output_empty_text(self):
self.assertRaises(AttributeError, lambda: Shell.check_output(''))
def test_check_output_empty_text(self):
self.assertRaises(AttributeError, lambda: Shell.check_output(''))
def test_check_output_echo_num_0(self):
self.assertEqual(Shell.check_output('echo 0'), '0')
def test_check_output_echo_num_1(self):
self.assertEqual(Shell.check_output('echo 1'), '1')
def predict(self, X, class_name=None, method_name=None, tnp_dir='tmp',
keep_tmp_dir=False, num_format=lambda x: str(x)):
"""
Predict using the transpiled model.
Parameters
----------
:param X : {array-like}, shape (n_features) or (n_samples, n_features)
The input data.
:param class_name : string, default: None
The name for the ported class.
:param method_name : string, default: None
The name for the ported method.
:param tnp_dir : string, default: 'tmp'
The path to the temporary directory for
storing the transpiled (and compiled) model.
:param keep_tmp_dir : bool, default: False
Whether to delete the temporary directory
or not.
:param num_format : lambda x, default: lambda x: str(x)
The representation of the floating-point values.
Returns
-------
y : int or array-like, shape (n_samples,)
The predicted class or classes.
"""
if class_name is None:
class_name = self.estimator_name
if method_name is None:
method_name = self.target_method
# Dependencies:
if not self._tested_dependencies:
self._test_dependencies()
self._tested_dependencies = True
# Support:
if 'predict' not in set(self.template.SUPPORTED_METHODS):
error = "Currently the given model method" \
" '{}' isn't supported.".format('predict')
raise AttributeError(error)
# Cleanup:
Shell.call('rm -rf {}'.format(tnp_dir))
Shell.call('mkdir {}'.format(tnp_dir))
# Transpiled model:
details = self.export(class_name=class_name,
method_name=method_name,
num_format=num_format,
details=True)
filename = Porter._get_filename(class_name, self.target_language)
target_file = os.path.join(tnp_dir, filename)
with open(target_file, str('w')) as file_:
file_.write(details.get('estimator'))
# Compilation command:
comp_cmd = details.get('cmd').get('compilation')
if comp_cmd is not None:
Shell.call(comp_cmd, cwd=tnp_dir)
# Execution command:
exec_cmd = details.get('cmd').get('execution')
exec_cmd = str(exec_cmd).split()
pred_y = None
# Single feature set:
if exec_cmd is not None and len(X.shape) == 1:
full_exec_cmd = exec_cmd + [str(sample).strip() for sample in X]
pred_y = Shell.check_output(full_exec_cmd, cwd=tnp_dir)
pred_y = int(pred_y)
# Multiple feature sets:
if exec_cmd is not None and len(X.shape) > 1:
pred_y = np.empty(X.shape[0], dtype=int)
for idx, features in enumerate(X):
full_exec_cmd = exec_cmd + [str(f).strip() for f in features]
pred = Shell.check_output(full_exec_cmd, cwd=tnp_dir)
pred_y[idx] = int(pred)
# Cleanup:
if not keep_tmp_dir:
Shell.call('rm -rf {}'.format(tnp_dir))
return pred_y