def test_with(self):
c = Chain([
Add(13),
])
data = {"input": 10}
with c:
c.process(data)
assert data == {"input": 10, "output": 23}
def test_chain_from_fuction(self):
data = {"input": 10}
def add(x):
x["output"] = x["input"] + 13
c = Chain(add)
c.process(data)
self.assertEqual(data, {"input": 10, "output": 23})
assert str(c) == "Chain[add]"
def create_navigation(self, navigation_html, experiment_folder,
experiment_path, data):
if experiment_folder != "":
if os.path.exists(experiment_path + "/custom_flexp_chain.py"):
try:
custom_flexp_chain = import_by_filename(
'custom_flexp_chain',
experiment_path + "/custom_flexp_chain.py")
html_chain = custom_flexp_chain.get_chain()
html_chain = Chain(html_chain)
except:
html_chain = Chain([
StringToHtml(
"<h2>Error processing custom chain. {}</h2>".
format(traceback.format_exc().replace(
"\n", "</br>")),
title="Error in custom chain")
] + self.html_chain.modules)
finally:
if "custom_flexp_chain" in sys.modules:
del sys.modules["custom_flexp_chain"]
else:
html_chain = self.html_chain
html_chain.process(data)
html_chain.close()
navigation_html = html_anchor_navigation(
experiment_path, experiment_folder,
html_chain) + navigation_html
return navigation_html
def test_chain(self):
data = {"input": 10}
c = Chain([
Add(13),
])
c.process(data)
c.close()
self.assertEqual(data, {"input": 10, "output": 23})
assert str(c) == "Chain[Add]"
c.add(Mult(2))
assert str(c) == "Chain[Add-Mult]"
c.process(data)
self.assertEqual(data, {"input": 10, "output": 20})
def test_chain_requires_first(self):
self.assertRaises(
KeyError, lambda: Chain([
Add(13),
],
check=True,
ignore_first_module_requirements=False))
def __init__(self, chain=None, max_processes=5, copy_chain=False):
"""
Sets up chain
@param copy_chain: bool If true, the chain will be copy.copyied
"""
self.chain = Chain(chain)
self.max_processes = max_processes
self.copy_chain = copy_chain
def test_time(self):
data = {"input": 10}
c = Chain([
Add(13),
])
c.process(data)
c.close()
assert len(c.times) == 1
assert c.iterations == 1
c.process(data)
assert c.iterations == 2
def run(port=7777,
chain=default_html_chain,
get_metrics_fcn=default_get_metrics,
metrics_file="metrics.csv",
additional_paths=None):
"""
Run the whole browser with optional own `port` number and `chain` of ToHtml modules.
Allows reading main metrics from all experiments and show them in experiment list.
:param list[tuple(str, StaticFileHandler, dict[str, str])] additional_paths: list of paths tha should be added
to tornado Application
:param int port: Port on which to start flexp browser
:param list[ToHtml]|ToHtml chain: List of ToHtml instances that defines what to print
:param (Callable[str]) -> list[dict[str, Any]] get_metrics_fcn: Function that takes filename of a file with
metrics and return dict[metric_name, value].
:param str metrics_file: Filename in each experiment dir which contains metrics values.
"""
# append new modules to the default chain
if isinstance(chain, ToHtml):
chain = [chain]
# handle wrong return type and expetions in get_metrics_fcn
get_metrics_fcn = return_type_list_of_dicts(get_metrics_fcn,
return_on_fail=[{}])
get_metrics_fcn = exception_safe(get_metrics_fcn, return_on_exception=[{}])
main_handler_params = {
"get_metrics_fcn": get_metrics_fcn,
"metrics_file": metrics_file,
"experiments_folder": os.getcwd(),
"html_chain": Chain(chain),
}
here_path = os.path.dirname(os.path.abspath(__file__))
additional_paths = additional_paths if additional_paths else []
app = tornado.web.Application(
[(r"/", MainHandler, main_handler_params),
(r'/(favicon.ico)', tornado.web.StaticFileHandler, {
"path": path.join(here_path, "static/")
}), (r"/file/(.*)", NoCacheStaticHandler, {
'path': os.getcwd()
}),
(r"/static/(.*)", tornado.web.StaticFileHandler, {
'path': path.join(here_path, "static")
}), (r"/ajax", AjaxHandler, {
"experiments_folder": os.getcwd()
})] + additional_paths, {"debug": True})
app.listen(port)
log.info("Starting server on port {:d}".format(port))
tornado.ioloop.IOLoop.current().start()
def main(port):
chain = [
CsvToHtml(file_name_pattern="worst_examples_.*.csv",
title="Worst examples",
delimiter='\t'),
]
add_paths = [(r"/features/", FeaturesHandler, {
"experiments_folder": os.getcwd(),
"html_chain": Chain(chain)
})]
browser.run(port=port, chain=chain, additional_paths=add_paths)
def test_chain_inspect_deep(self):
data = {"input": {i: i for i in range(11)}}
with LogCapture() as l:
c = Chain([
inspector.inspect(DummyModule(), stream=True)])
c.process(data)
c.close()
l.check(
('flexp.flow.flow', 'DEBUG', 'DummyModule.process()'),
('flexp.flow.inspector', 'INFO', 'Data flow structure'),
('flexp.flow.inspector', 'INFO', "{\'input\': {\"<class \'int\'>#11 times (0)\": 0}}"),
('flexp.flow.inspector', 'INFO', 'End of data flow structure'),
('flexp.flow.flow', 'INFO', 'DummyModule average execution time 0.00 sec')
)
def test_chain_inspect(self):
data = {"input": 20}
with LogCapture() as l:
c = Chain([
inspector.inspect(Add(10), stream=True)])
c.process(data)
c.close()
l.check(
('flexp.flow.flow', 'DEBUG', 'Add.process()'),
('flexp.flow.inspector', 'INFO', 'Data flow structure'),
('flexp.flow.inspector', 'INFO', "{'input': 20, 'output': 30}"),
('flexp.flow.inspector', 'INFO', 'End of data flow structure'),
('flexp.flow.flow', 'INFO', 'Add average execution time 0.00 sec')
)
def test_chain_inherits_provides_requires(self):
data = {"input": 20}
Chain([Chain([Add(10)], check=True),
Result()], check=True).process(data)
assert data["output"] == 30
def get(self):
experiment_folder = self.get_argument("experiment", default="")
experiment_path = path.join(self.experiments_folder, experiment_folder)
if not path.isdir(experiment_path):
experiment_folder = ""
navigation_html = html_navigation(self.experiments_folder, experiment_folder)
header_html = ""
scripts_html = ""
if experiment_folder != "":
data = {"experiment_path": experiment_path,
"experiment_folder": experiment_folder,
"html": [],
"header": dict(),
"scripts": dict()
}
# Use custom chain, if present
if os.path.exists(experiment_path + "/custom_flexp_chain.py"):
try:
custom_flexp_chain = import_by_filename('custom_flexp_chain',
experiment_path + "/custom_flexp_chain.py")
html_chain = custom_flexp_chain.get_chain()
html_chain = Chain(html_chain)
except:
html_chain = Chain([StringToHtml("<h2>Error processing custom chain. {}</h2>"
.format(traceback.format_exc().replace("\n", "</br>")),
title="Error in custom chain")] + self.html_chain.modules)
finally:
if "custom_flexp_chain" in sys.modules:
del sys.modules["custom_flexp_chain"]
else:
html_chain = self.html_chain
html_chain.process(data)
html_chain.close()
title_html = "<h1>{}</h1>".format(experiment_folder)
content_html = u"\n".join(data['html'])
navigation_html = html_anchor_navigation(
experiment_path, experiment_folder, html_chain) + navigation_html
header_html = u"\n".join(u"\n".join(html_lines)
for head_section, html_lines
in data["header"].items())
scripts_html = u"\n".join(u"\n".join(script_lines)
for script_section, script_lines
in data["scripts"].items())
else:
title_html = "<h1>Experiments</h1>"
content_html = html_table(self.experiments_folder, self.get_metrics_fcn, self.metrics_file)
html = self._template.format(
title=title_html,
navigation=navigation_html,
content=content_html,
header=header_html,
scripts=scripts_html)
self.write(html)
def test_chain_requires(self):
self.assertRaises(
KeyError, lambda: Chain([Add(13), RequiresNonsense()], check=True))
# Store the running program
flexp.backup_files([sys.argv[0], "simple_modules.py"])
# Store all files in these directories into a zip file
flexp.backup_sources(["../flexp/"])
flexp.describe("Query parameter prediction with TF-IDF and linear regression")
# Setup logging
log.debug("flow setup complete.")
# Create the chain to load the data, lowercase and tokenize it.
file_name = "example_queries.tsv"
data_chain = Chain([
LoadData(file_name),
Lowercase(),
TfIdf(),
])
# data["id"] should contain all info required to replicate the experiment.
data = {"id": file_name}
data_chain.process(data)
data_chain.close()
log.debug("Data chain complete.")
# Train and evaluate a classifier
train_chain = Chain([
# Create data["train"] and data["dev"]
TrainTestSplit(),
# Train our classifier on data["train"]
Train(),
def test_chain_names(self):
assert str(Chain()) == "Chain[]"
assert str(Chain(name="ML-pipeline")) == "ML-pipeline[]"
def test_chain_from_object(self):
data = {"input": 10}
c = Chain(Add(13))
c.process(data)
self.assertEqual(data, {"input": 10, "output": 23})
assert str(c) == "Chain[Add]"
