def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
if not os.path.exists(doc):
raise FileNotFoundError(doc)
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
import pyquickhelper
import jyquickhelper
import pyensae
import ensae_teaching_cs
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs])
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=ensae_teaching_cs)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
this = os.path.join(temp, "..", "data",
"fr.openfoodfacts.org.products.head100.csv")
this = os.path.normpath(this)
def clean_function(cell):
cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv",
this.replace("\\", "/"))
return cell
import pyquickhelper
import jyquickhelper
import pyensae
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, thismodule])
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid,
clean_function=clean_function)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def a_test_notebook_runner(self,
name,
folder,
valid=None,
copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
import pyquickhelper
import jyquickhelper
import pyensae
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, thismodule])
res = execute_notebook_list(temp,
keepnote,
additional_path=add_path,
valid=valid)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_doc_page(self):
temp = get_temp_folder(__file__, "temp_doc_page")
preamble = TestDocPage.preamble + TestDocPage.custom_preamble
this = os.path.abspath(os.path.dirname(__file__))
root = os.path.join(this, "..", "..", "_doc", "sphinxdoc", "source",
"c_ml")
image_path = "piecewise"
rst = os.path.join(root, "piecewise.rst")
imgs = os.path.join(root, image_path)
content = self.read_file(rst)
synchronize_folder(imgs,
os.path.join(temp, image_path),
create_dest=True)
epkg_dictionary = {
'XD':
'http://www.xavierdupre.fr',
'scikit-learn':
'https://scikit-learn.org/stable/',
'sklearn':
('http://scikit-learn.org/stable/',
('http://scikit-learn.org/stable/modules/generated/{0}.html', 1),
('http://scikit-learn.org/stable/modules/generated/{0}.{1}.html',
2)),
'ICML 2016':
'link',
}
writer = 'html'
ht = rst2html(content,
writer=writer,
layout="sphinx",
keep_warnings=True,
imgmath_latex_preamble=preamble,
outdir=temp,
epkg_dictionary=epkg_dictionary)
ht = ht.replace('src="_images/', 'src="')
ht = ht.replace('/scripts\\bokeh', '../bokeh_plot\\bokeh')
ht = ht.replace('/scripts/bokeh', '../bokeh_plot/bokeh')
rst = os.path.join(temp, "out.{0}".format(writer))
self.write_file(rst, ht)
ht = ht.split('<div class="section" id="notebooks">')[0]
# Tests the content.
self.assertNotIn('runpythonerror', ht)
lines = ht.split('\n')
for i, line in enumerate(lines):
if 'WARNING' in line:
if "contains reference to nonexisting document" in lines[i +
1]:
continue
else:
mes = 'WARNING issue\n File "{0}", line {1}'.format(
rst, i + 1)
raise Exception(mes)
def test_synchronize(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_synchronize")
data = os.path.join(temp, '..', 'data')
synchronize_folder(data, temp, fLOG=fLOG)
self.assertExists(os.path.join(temp, "loghelper.zip"))
dest = os.path.join(temp, "dest")
self.assertRaise(lambda: synchronize_folder(data, dest, fLOG=fLOG),
PQHException)
synchronize_folder(data, dest, fLOG=fLOG, create_dest=True)
self.assertExists(os.path.join(dest, "loghelper.zip"))
def test_synchronize(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_synchronize")
data = os.path.join(temp, '..', 'data')
synchronize_folder(data, temp, fLOG=fLOG)
self.assertExists(os.path.join(temp, "loghelper.zip"))
dest = os.path.join(temp, "dest")
self.assertRaise(lambda: synchronize_folder(
data, dest, fLOG=fLOG), PQHException)
synchronize_folder(data, dest, fLOG=fLOG, create_dest=True)
self.assertExists(os.path.join(dest, "loghelper.zip"))
def copy_tutorial(name, destination):
"""
copy files and scripts for a specific tutorial
@param name tutorial name or folder
@param destination destination
@return list of operations done by the function list of 3-uple: action, source_file, dest_file
The function will create a sub folder in *destination*
using *name* or the last folder name in *name*.
This function requires modules
`pyquickhelper <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/>`_.
"""
from pyquickhelper.filehelper import synchronize_folder
if os.path.exists(name):
dest = os.path.join(destination, name)
else:
this = os.path.abspath(os.path.dirname(__file__))
fold = os.path.join(this, name)
if not os.path.exists(fold):
raise FileNotFoundError("unable to find tutorial {0}\n{1} not here".format(name, fold))
spl = os.path.split(fold)
dest = os.path.join(destination, spl[-1])
if not os.path.exists(dest):
os.mkdir(dest)
return synchronize_folder(fold, dest)
def copy_tutorial(name, destination):
"""
copy files and scripts for a specific tutorial
@param name tutorial name or folder
@param destination destination
@return list of operations done by the function list of 3-uple: action, source_file, dest_file
The function will create a sub folder in *destination*
using *name* or the last folder name in *name*.
This function requires modules
`pyquickhelper <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/>`_.
"""
from pyquickhelper.filehelper import synchronize_folder
if os.path.exists(name):
dest = os.path.join(destination, name)
else:
this = os.path.abspath(os.path.dirname(__file__))
fold = os.path.join(this, name)
if not os.path.exists(fold):
raise FileNotFoundError(
"unable to find tutorial {0}\n{1} not here".format(name, fold))
spl = os.path.split(fold)
dest = os.path.join(destination, spl[-1])
if not os.path.exists(dest):
os.mkdir(dest)
return synchronize_folder(fold, dest)
def test_doc_page(self):
temp = get_temp_folder(__file__, "temp_doc_page")
preamble = TestDocPage.preamble + TestDocPage.custom_preamble
this = os.path.abspath(os.path.dirname(__file__))
root = os.path.join(this, "..", "..", "_doc",
"sphinxdoc", "source", "c_ml")
image_path = "piecewise"
rst = os.path.join(root, "piecewise.rst")
imgs = os.path.join(root, image_path)
content = self.read_file(rst)
synchronize_folder(imgs, os.path.join(
temp, image_path), create_dest=True)
epkg_dictionary = {
'XD': 'http://www.xavierdupre.fr',
'scikit-learn': 'https://scikit-learn.org/stable/',
'sklearn': ('http://scikit-learn.org/stable/',
('http://scikit-learn.org/stable/modules/generated/{0}.html', 1),
('http://scikit-learn.org/stable/modules/generated/{0}.{1}.html', 2)),
'ICML 2016': 'link',
}
writer = 'html'
ht = rst2html(content, writer=writer, layout="sphinx", keep_warnings=True,
imgmath_latex_preamble=preamble, outdir=temp,
epkg_dictionary=epkg_dictionary)
ht = ht.replace('src="_images/', 'src="')
ht = ht.replace('/scripts\\bokeh', '../bokeh_plot\\bokeh')
ht = ht.replace('/scripts/bokeh', '../bokeh_plot/bokeh')
rst = os.path.join(temp, "out.{0}".format(writer))
self.write_file(rst, ht)
ht = ht.split('<div class="section" id="notebooks">')[0]
# Tests the content.
self.assertNotIn('runpythonerror', ht)
lines = ht.split('\n')
for i, line in enumerate(lines):
if 'WARNING' in line:
if "contains reference to nonexisting document" in lines[i + 1]:
continue
else:
mes = 'WARNING issue\n File "{0}", line {1}'.format(
rst, i + 1)
raise Exception(mes)
def test_patch(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
# not setup for python 2.7
return
pattern = "([#][!](([A-Za-z][:])?[/\\\\]?[-a-zA-Z0-9_.]+[/\\\\])*pythonw?[.]exe)"
reg_exe = re.compile(pattern, re.IGNORECASE)
text = "#!C:\\github\\pymyinstall\\dist\\win_python_setup\\python\\python.exe"
assert len(reg_exe.findall(text)) > 0
temp = get_temp_folder(__file__, "temp_patch")
data = os.path.join(temp, "..", "data", "Scripts")
dest = os.path.join(temp, "Scripts")
if not os.path.exists(dest):
os.mkdir(dest)
if not os.path.exists(data):
warnings.warn(
"pyquickhelper forgot to copy this folder for python 2.7")
raise FileNotFoundError(data)
synchronize_folder(data, dest)
op = win_patch_paths(dest, "", fLOG=fLOG)
into = "#!python.exe"
binto = bytes(into, encoding="ascii")
assert len(op) > 0
i = 0
for _, full in op:
with open(full, "rb") as f:
content = f.read()
if binto not in content:
raise Exception("file:{}\ncontent:\n{}".format(full, content))
i += 1
exp = 3
if i != exp:
raise Exception("i != " + str(exp) + "\n" + str(op))
op = win_patch_paths(dest, None, fLOG=fLOG)
def copy_jstool2notebook(tool, force=False):
"""
Copies a tool to :epkg:`notebook` folder.
@param tool tool name (snap for example)
@param force do the copy even if the destination folder exists
@return list of copied files
"""
import notebook
dest = os.path.join(os.path.dirname(notebook.__file__), "static")
src = os.path.join(os.path.dirname(__file__), tool)
if not os.path.exists(src):
raise FileNotFoundError("unable to find tool: " + tool)
dest = os.path.join(dest, tool)
if not os.path.exists(dest):
os.mkdir(dest)
return synchronize_folder(src, dest)
elif force:
return synchronize_folder(src, dest)
else:
return None
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
this = os.path.join(temp, "..", "data",
"fr.openfoodfacts.org.products.head100.csv")
this = os.path.normpath(this)
def clean_function(cell):
cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv",
this.replace("\\", "/"))
return cell
import pyquickhelper
import jyquickhelper
import pyensae
import ensae_teaching_cs
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs])
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid,
clean_function=clean_function)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=ensae_teaching_cs)
"""
===================
Syncing two folders
===================
This example is about syncing two folders
or creating a backup.
"""
###############################
from pyquickhelper.filehelper import synchronize_folder
###############################
dest = "temp_sync"
import os
if not os.path.exists(dest):
print("creating", dest)
os.makedirs(dest)
else:
print("folder already created", dest)
###############################
synchronize_folder(os.path.abspath("."),
dest, fLOG=print)
if "_UT_" in spl:
parent, spl = os.path.split(parent)
if "_UT_" in spl:
raise ValueError("Something is weird with: '{0}'".format(name))
index.append((spl, os.path.dirname(name)))
if spl in done:
raise ValueError("Duplicated package '{0}'.\n{1}".format(
spl, "\n".join("{0}={1}".format(k, v)
for k, v in sorted(done.items()))))
fLOG("Found {0} directories".format(len(index)))
for ind in index:
fLOG(" ", ind)
########################################
# copies
for pack, folder in index:
fLOG("########################")
fLOG("Copy of", pack)
fLOG("########################")
to = os.path.join(dest, pack)
fLOG("destination", to)
if not os.path.exists(to):
os.makedirs(to)
status_file = os.path.join(dest, pack + ".txt")
synchronize_folder(folder,
to,
log1=True,
copy_1to2=True,
file_date=status_file)
for name in explore_folder_iterfile(root, pattern):
if name.endswith(sub):
pack = name[:len(name) - len(sub) - 1]
parent, spl = os.path.split(pack)
if "_UT_" in spl:
parent, spl = os.path.split(parent)
if "_UT_" in spl:
raise ValueError("Something is weird with: '{0}'".format(name))
index.append((spl, os.path.dirname(name)))
if spl in done:
raise ValueError("Duplicated package '{0}'.\n{1}".format(
spl, "\n".join("{0}={1}".format(k, v) for k, v in sorted(done.items()))))
fLOG("Found {0} directories".format(len(index)))
for ind in index:
fLOG(" ", ind)
########################################
# copies
for pack, folder in index:
fLOG("########################")
fLOG("Copy of", pack)
fLOG("########################")
to = os.path.join(dest, pack)
fLOG("destination", to)
if not os.path.exists(to):
os.makedirs(to)
status_file = os.path.join(dest, pack + ".txt")
synchronize_folder(folder, to, log1=True, copy_1to2=True,
file_date=status_file)
from pyquickhelper.filehelper import synchronize_folder
###############################
dest = "temp_sync"
import os
if not os.path.exists(dest):
print("creating", dest)
os.makedirs(dest)
else:
print("folder already created", dest)
###############################
try:
copy = os.path.dirname(__file__)
except NameError:
# __file__ does not exist when run with sphinx-gallery
copy = "."
paths_to_try = [
os.path.join("..", "notebooks"),
os.path.join("_doc", "notebooks")
]
for path in paths_to_try:
src = os.path.abspath(path)
if os.path.exists(src):
break
src = os.path.abspath(src)
synchronize_folder(src, dest, fLOG=print)
def copy_assemblies(libdef=None, version="Release"):
"""
Copies all assemblies in the right location.
*libdef* can be ``None``, ``ml`` or ``mlext``.
"""
from pyquickhelper.filehelper import synchronize_folder
if libdef == 'ml':
folders = []
copy2 = True
for lib in [
"Microsoft.ML.Api",
"Microsoft.ML.Console",
# "Microsoft.ML.DnnAnalyzer",
# "Microsoft.ML.Ensemble",
"Microsoft.ML.FastTree",
# "Microsoft.ML.HalLearners",
"Microsoft.ML.ImageAnalytics",
"Microsoft.ML.KMeansClustering",
# "Microsoft.ML.Legacy",
"Microsoft.ML.LightGBM",
"Microsoft.ML.Maml",
# "Microsoft.ML.Onnx",
# "Microsoft.ML.OnnxTransform",
# "Microsoft.ML.PCA",
# "Microsoft.ML.PipelineInference",
"Microsoft.ML.Sweeper",
"Microsoft.ML.TensorFlow",
# "Microsoft.ML.Transforms",
# "Microsoft.ML.StandardLearners",
"Microsoft.ML.TimeSeries",
]:
fold = 'cscode/machinelearning/bin/AnyCPU.%s/%s' % (version, lib)
if not os.path.exists(fold):
# To avoid copy, we check that machinelearningext is
# present at the same level as csharyml.
this = os.path.normpath(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..')))
mlext = os.path.join(this, 'machinelearningext')
if not os.path.exists(mlext):
raise FileNotFoundError(
"Unable to find folder '{0}' or '{1}'.".format(
fold, mlext))
fold = os.path.join(mlext, "machinelearning",
'bin/AnyCPU.%s/%s' % (version, lib))
if not os.path.exists(mlext):
raise FileNotFoundError(
"Unable to find folder '{0}' or '{1}'.".format(
fold, fold))
copy2 = False
folders.append(fold)
dests = ['cscode/bin/machinelearning/%s' % version]
if copy2:
dests.append('cscode/machinelearningext/machinelearning/dist/%s' %
version)
elif libdef == 'mlext':
folders = []
for sub in [
'DataManipulation',
'DocHelperMlExt',
'ScikitAPI',
]:
fold = 'cscode/machinelearningext/machinelearningext/bin/AnyCPU.%s/%s/netstandard2.0' % (
version, sub)
if not os.path.exists(fold):
# To avoid copy, we check that machinelearningext is
# present at the same level as csharyml.
this = os.path.normpath(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..')))
mlext = os.path.join(this, 'machinelearningext')
if not os.path.exists(mlext):
raise FileNotFoundError(
"Unable to find folder '{0}' or '{1}'.".format(
fold, mlext))
fold = os.path.join(mlext, "machinelearningext",
'bin/AnyCPU.%s/%s' % (version, sub))
if not os.path.exists(mlext):
raise FileNotFoundError(
"Unable to find folder '{0}' or '{1}'.".format(
fold, fold))
copy2 = False
folders.append(fold)
dests = [
'cscode/bin/machinelearningext/%s' % version,
]
else:
folders = [
'cscode/bin/machinelearning/%s' % version,
'cscode/bin/machinelearningext/%s' % version,
'cscode/bin/AnyCPU.%s/CSharPyMLExtension/netstandard2.0' % version
]
rootpkg = "cscode/machinelearning/packages"
if not os.path.exists(rootpkg):
# To avoid copy, we check that machinelearningext is
# present at the same level as csharyml.
this = os.path.normpath(
os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
mlext = os.path.join(this, 'machinelearningext')
if not os.path.exists(mlext):
raise FileNotFoundError(
"Unable to find folder '{0}' or '{1}'.".format(
fold, mlext))
rootpkg = os.path.join(mlext, "machinelearning", "packages")
folders.extend([
os.path.join(rootpkg, "newtonsoft.json", "10.0.3", "lib",
"netstandard1.3"),
os.path.join(rootpkg, "system.memory", "4.5.1", "lib",
"netstandard2.0"),
os.path.join(rootpkg, "system.runtime.compilerservices.unsafe",
"4.5.0", "lib", "netstandard2.0"),
os.path.join(rootpkg, "system.collections.immutable", "1.5.0",
"lib", "netstandard2.0"),
os.path.join(rootpkg, "system.numerics.vectors", "4.4.0", "lib",
"netstandard2.0"),
os.path.join(rootpkg, "lightgbm", "2.2.1.1", "runtimes", "win-x64",
"native"),
os.path.join(rootpkg, "google.protobuf", "3.5.1", "lib",
"netstandard1.0"),
])
if sys.platform.startswith("win"):
sub = "win-x64"
else:
sub = "linux-x64"
# Additional dependencies.
folders.extend([
os.path.join(rootpkg, "system.threading.tasks.dataflow", "4.8.0",
"lib", "netstandard2.0"),
os.path.join(rootpkg, "system.drawing.common", "4.5.0", "lib",
"netstandard2.0"),
os.path.join(rootpkg, "mlnetmkldeps", "0.0.0.7", "runtimes", sub,
"native"),
])
dests = ['src/csharpyml/binaries/%s' % version]
for dest in dests:
if not os.path.exists(dest):
os.makedirs(dest)
if libdef is None:
init = os.path.join(dest, '__init__.py')
if not os.path.exists(init):
with open(init, 'w') as f:
pass
for fold in folders:
try:
v, n, found = find_folder_package(fold)
do_check = True
except FileNotFoundError:
found = fold
do_check = False
if do_check:
if "packages" in fold:
if v is None:
raise FileNotFoundError(
"Unable to find a suitable version for package '{0}'"
.format(fold))
elif 'Native' not in found and 'netcoreapp' not in found and 'netstandard' not in found:
raise FileNotFoundError(
"Unable to find a suitable folder binaries '{0}'".
format(fold))
print("[csharpyml.copy] '{0}' -> '{1}'".format(found, dest))
synchronize_folder(found, dest, fLOG=print, no_deletion=True)
if libdef not in ('ml', 'mlext'):
if sys.platform.startswith("win"):
check_existence = "src/csharpyml/binaries/%s/System.Numerics.Vectors.dll" % version
else:
check_existence = "src/csharpyml/binaries/%s/System.Numerics.Vectors.dll" % version
if not os.path.exists(check_existence):
found = "\n".join(os.listdir(os.path.dirname(check_existence)))
warnings.warn("Unable to find '{0}', found:\n{1}".format(
check_existence, found))