def save_tags(self, n_clicks, tags): if not n_clicks or not self.s.selected or not len(self.s.selected) == 1: raise PreventUpdate es = ExperimentService(Path(self.s.selected[0]["path"])) es.set_tags(tags) self.s.selected[0]["tags"] = " ".join(tags) all_tags = get_all_tags() if any(tag not in all_tags for tag in tags): # Need to clear the memoized cache for `get_all_tags` now. cache.delete_memoized(get_all_tags) return True, None
def re_build_database(n_clicks): if not n_clicks: raise PreventUpdate start_time = time.time() # Remove any deleted experiments, track new experiments added. ExperimentService.init_db_table() # Clear the memoized cache for `get_all_tags` now. cache.delete_memoized(get_all_tags) # Ensure this takes at least 1 second so that the spinner notification has # time to display (it looks cool). if (time.time() - start_time) < 1: time.sleep(1) return False, True, None
def serve(config, launch, skip_db_update): """ Serve the dashboard locally. """ monkey.patch_all() from mallennlp.app import create_app from mallennlp.services.db import get_db_from_cli, init_tables, Tables from mallennlp.services.experiment import ExperimentService if not skip_db_update: db = get_db_from_cli(config) click.echo("Updating experiments index...") init_tables(db, (Tables.EXPERIMENTS.value, )) entries = [ s.get_db_fields() for s in ExperimentService.find_experiments() ] if entries: click.echo( f"Found {click.style(str(len(entries)), fg='green')} experiments" ) ExperimentService.init_db_table(db=db, entries=entries) else: click.secho("No experiments found", fg="yellow") db.close() click.secho( f"Serving AllenNLP manager for {click.style(config.project.name, bold=True)}", fg="green", ) application = create_app(config) if launch: url = f"http://localhost:{config.server.port}" click.launch(url) options = { "timeout": 300, "workers": config.server.workers, "worker_class": "gevent", "worker_connections": config.server.worker_connections, "bind": f":{config.server.port}", "loglevel": config.project.loglevel.lower(), "preload_app": True, } StandaloneApplication(application, options).run()
def check_path_valid(self, attribute, value): try: path = ExperimentService.get_canonical_path(Path(value)) if not path.exists(): raise InvalidPageParametersError( "Directory does not exist.") if path.is_dir() and not ExperimentService.is_experiment(path): raise InvalidPageParametersError( "Directory is not an experiment.") if path.is_file(): raise InvalidPageParametersError( "Path is a file, not an experiment directory.") except ValueError: raise InvalidPageParametersError( "Directory outside of project.") return str(path)
def test_set_tags(experiment_service, db): experiment_service.set_tags(["copynet", "seq2seq"]) assert experiment_service.get_tags() == ["copynet", "seq2seq"] # Should be written to file now. assert experiment_service.e.meta.path.exists() es2 = ExperimentService(experiment_service.get_path()) assert es2.get_tags() == ["copynet", "seq2seq"] # Should be updated in database as well. results = list( db.execute( f"SELECT * FROM {Tables.EXPERIMENTS.value} WHERE path = ?", (str(experiment_service.get_path()), ), )) assert len(results) == 1 assert results[0]["tags"] == "copynet seq2seq"
def reset(db, table: str): click.echo(f"Resetting db table {click.style(table, fg='green')}...") from mallennlp.services.db import init_tables, Tables from mallennlp.services.experiment import ExperimentService init_tables(db, (table, )) if table == Tables.EXPERIMENTS.value: # Recursively search project for experiments and populate database. entries = [ s.get_db_fields() for s in ExperimentService.find_experiments() ] if entries: click.echo( f"Found {click.style(str(len(entries)), fg='green')} experiments" ) ExperimentService.init_db_table(db=db, entries=entries) db.close() click.secho("Success!", fg="green")
def display_tags(es: ExperimentService): tags = es.get_tags() tag_badges = [ dbc.Badge( [t, html.I(className="fas fa-times-circle", style={"margin-left": "5px"})], id=f"experiment-tag-{i}", key=t, color="info", className="mr-1", pill=True, href="#", ) for i, t in enumerate(tags[:MAX_TAG_BADGES]) ] tooltips = [ dbc.Tooltip(f"""Remove "{t}" tag""", target=f"experiment-tag-{i}") for i, t in enumerate(tags[:MAX_TAG_BADGES]) ] for i in range(len(tags), MAX_TAG_BADGES): tag_badges.append( dbc.Badge("", id=f"experiment-tag-{i}", href="#", style={"display": "none"}) ) if len(tags) > MAX_TAG_BADGES: tag_badges.append( dbc.Badge( html.I(className="fas fa-ellipsis-h"), id="experiment-edit-tags-modal-open", color="info", className="mr-1", pill=True, href="#", ) ) tooltips.append( dbc.Tooltip( "See all tags or edit", target="experiment-edit-tags-modal-open" ) ) else: tag_badges.append( dbc.Badge( html.I(className="fas fa-edit", style={"margin-left": "3px"}), id="experiment-edit-tags-modal-open", color="info", className="mr-1", pill=True, href="#", ) ) tooltips.append( dbc.Tooltip("Edit tags", target="experiment-edit-tags-modal-open") ) return [html.Span(tag_badges), html.Div(children=tooltips)]
def display_metrics(es: ExperimentService, epoch: Optional[int] = None): metrics = es.get_metrics(epoch) if metrics is None: return dcc.Markdown("**No metrics to display**") fields: List[str] = [] for field_name, formatter in METRIC_EPOCH_DISPLAY_FIELDS.items(): field_value = metrics.get(field_name) if field_value is not None: fields.append(formatter % field_value) for other_field_name in metrics: if other_field_name in METRIC_EPOCH_DISPLAY_FIELDS: continue if other_field_name.startswith("best_validation_"): field_value = metrics[other_field_name] fields.append( f"**Best epoch validation {other_field_name[16:].replace('_', ' ')}:** `{field_value}`" ) return dcc.Markdown("\n\n".join(fields))
class ExperimentPage(Page): @serde class Params: path: str = attr.ib() active: str = "overview" @path.validator def check_path_valid(self, attribute, value): try: path = ExperimentService.get_canonical_path(Path(value)) if not path.exists(): raise InvalidPageParametersError( "Directory does not exist.") if path.is_dir() and not ExperimentService.is_experiment(path): raise InvalidPageParametersError( "Directory is not an experiment.") if path.is_file(): raise InvalidPageParametersError( "Path is a file, not an experiment directory.") except ValueError: raise InvalidPageParametersError( "Directory outside of project.") return str(path) def __init__(self, state, params): super().__init__(state, params) self.path = ExperimentService.get_canonical_path(Path(self.p.path)) self.es = ExperimentService(self.path) def get_experiment_header_elements(self): status = ec.get_status(self.es) elements = [ html.H5([ *ec.get_path_breadcrumbs(self.path), ec.get_status_badge(status) ]), html.Div(id="experiment-tags", children=ec.display_tags(self.es)), ec.edit_tags_modal("experiment"), ] if status not in (Status.FINISHED, Status.FAILED): interval = 1000 * 30 else: interval = 1000 * 300 elements.append( dcc.Interval(id="experiment-update-interval", interval=interval)) return elements def get_overview_elements(self): out = [ html.Div(id="experiment-overview", children=ec.display_metrics(self.es)), html.Strong("Logs: "), dcc.Link( "STDOUT", href="/log-stream?" + urlparse.urlencode({"path": self.path / self.es.STDOUT_FNAME}), ), ", ", dcc.Link( "STDERR", href="/log-stream?" + urlparse.urlencode({"path": self.path / self.es.STDERR_FNAME}), ), ] epochs = self.es.get_epochs() if epochs: out.extend([ html.Br(), html.Br(), dcc.Dropdown( id="experiment-metric-plot-dropdown", options=[{ "label": m, "value": m } for m in self.es.get_metric_names()], value="loss", ), dcc.Graph( id="experiment-metric-plot", config={"displayModeBar": False}, figure=ec.get_metric_plot_figure(self.es, "loss"), ), ]) return out def get_settings_elements(self): return ["Coming soon"] def wrap_elements(self, elements): return [ dbc.Row( dbc.Col( self.get_experiment_header_elements(), className="dash-padded-element experiment-header-element", )), dbc.Row( dbc.Col( elements, className= "dash-padded-element experiment-main-content-element", )), ] def get_elements(self): # Update database entry. self.es.update_db_entry() # Create sidebar entries. entries = OrderedDict([ ( "overview", SidebarEntry( "Overview", lambda: self.wrap_elements(self.get_overview_elements()), className="", ), ), ( "settings", SidebarEntry( "Settings", lambda: self.wrap_elements(self.get_settings_elements()), className="", ), ), ]) return SidebarLayout("Experiment", entries, self.p.active, self.p) def get_notifications(self): return [ dbc.Toast( "Tags successfully updated", id="experiment-edit-tags-noti", header="Success", dismissable=True, duration=4000, is_open=False, icon="success", ) ] @Page.callback( [Output("experiment-metric-plot", "figure")], [ Input("experiment-metric-plot-dropdown", "value"), Input("experiment-update-interval", "n_intervals"), ], mutating=False, ) def update_metric_plot_figure(self, metric_name, n_intervals): if not metric_name and not n_intervals: raise PreventUpdate return ec.get_metric_plot_figure(self.es, metric_name) @Page.callback( [Output("experiment-status-badge", "children")], [Input("experiment-update-interval", "n_intervals")], mutating=False, ) def update_status_badge(self, _): # Update database entry. self.es.update_db_entry() status = ec.get_status(self.es) return ec.get_status_badge(status) @Page.callback( [Output("experiment-epoch-metrics", "children")], [Input("experiment-update-interval", "n_intervals")], mutating=False, ) def update_epoch_metrics(self, _): return ec.display_metrics(self.es) @Page.callback( [Output("experiment-tags", "children")], [ Input(f"experiment-tag-{i}", "n_clicks") for i in range(ec.MAX_TAG_BADGES) ] + [Input("experiment-edit-tags-noti", "is_open")], [ State(f"experiment-tag-{i}", "key") for i in range(ec.MAX_TAG_BADGES) ], mutating=False, permissions=Permissions.READ_WRITE, ) def update_tags(self, *args): ctx = dash.callback_context if not ctx.triggered: raise PreventUpdate button_id = ctx.triggered[0]["prop_id"].split(".")[0] if button_id.startswith("experiment-tag-"): # Try deleting tag. current_tags = self.es.get_tags() tag = ctx.states[f"{button_id}.key"] if tag in current_tags: self.es.set_tags([t for t in current_tags if t != tag]) # Need to clear the memoized cache for `get_all_tags` now. cache.delete_memoized(ec.get_all_tags) return ec.display_tags(self.es) @Page.callback( [ Output("experiment-edit-tags-modal", "is_open"), Output("experiment-edit-tags-dropdown", "value"), ], [ Input("experiment-edit-tags-modal-open", "n_clicks"), Input("experiment-edit-tags-modal-close", "n_clicks"), ], [State("experiment-edit-tags-modal", "is_open")], mutating=False, permissions=Permissions.READ_WRITE, ) def toggle_modal(self, n1, n2, is_open): tags: Optional[List[str]] = None if n1 or n2: will_open = not is_open if will_open and self.es: tags = self.es.get_tags() return will_open, tags return is_open, tags @staticmethod @Page.callback( [Output("experiment-edit-tags-dropdown", "options")], [Input("experiment-edit-tags-dropdown", "search_value")], [State("experiment-edit-tags-dropdown", "value")], ) def update_tag_options(search, value): if not search: raise PreventUpdate options = {t for t in ec.get_all_tags() if t.startswith(search)} options.add(search) for v in value or []: options.add(v) return [{"label": t, "value": t} for t in options] @Page.callback( [Output("experiment-edit-tags-noti", "is_open")], [Input("experiment-edit-tags-save", "n_clicks")], [State("experiment-edit-tags-dropdown", "value")], mutating=False, ) def save_tags(self, n_clicks, tags): if not n_clicks: raise PreventUpdate self.es.set_tags(tags) all_tags = ec.get_all_tags() if any(tag not in all_tags for tag in tags): # Need to clear the memoized cache for `get_all_tags` now. cache.delete_memoized(ec.get_all_tags) return True
def experiment_service(project, db): return ExperimentService(project / "test_experiment", db)
def test_add_experiments(db, entries): init_tables(db, (Tables.EXPERIMENTS.value, )) ExperimentService.init_db_table(db=db, entries=entries)
def test_get_canonical_path(path, root, result): assert str(ExperimentService.get_canonical_path(Path(path), Path(root))) == result
def test_remove_db_entry(experiment_service, db): ExperimentService.remove_db_entry(experiment_service.get_path(), db=db) assert len(list( db.execute(f"SELECT * FROM {Tables.EXPERIMENTS.value}"))) == 0
def test_find_experiments(project): exps = list(ExperimentService.find_experiments(project)) assert len(exps) == 1 assert exps[0].e.path == Path("test_experiment")
def __init__(self, state, params): super().__init__(state, params) self.path = ExperimentService.get_canonical_path(Path(self.p.path)) self.es = ExperimentService(self.path)
def init_project(verb: str, name: str, path: Path, username: str, password: str, **kwargs): from mallennlp.domain.config import ProjectConfig, ServerConfig from mallennlp.domain.user import Permissions from mallennlp.services.db import init_db, get_db_from_cli from mallennlp.services.experiment import ExperimentService from mallennlp.services.config import Config from mallennlp.services.user import UserService if (path / Config.CONFIG_PATH).exists(): raise click.ClickException( click.style("project already exists", fg="red")) # Initialize config. project_options = { k: v for k, v in kwargs.items() if not k.startswith("server_") and v is not None } server_options = { k[7:]: v for k, v in kwargs.items() if k.startswith("server_") and v is not None } config = Config( ProjectConfig(path, name=name, **project_options), ServerConfig(path, **server_options), ) # Ensure the server's instance path exists. os.makedirs(config.server.instance_path) # Initialize the database file. init_db(config) # Add the user to the database. db = get_db_from_cli(config) user_service = UserService(db=db) user_service.create(username, password, permissions=Permissions.ADMIN) # Find existing experiments and add to database (does nothing if new project). experiment_entries = [ s.get_db_fields() for s in ExperimentService.find_experiments() ] if experiment_entries: click.echo( f"Found {click.style(str(len(experiment_entries)), fg='green')} existing experiments" ) ExperimentService.init_db_table(db=db, entries=experiment_entries) # Save the config to the 'Project.toml' file in the project directory. config.to_toml(path) click.echo( f"{verb} project named {click.style(name, fg='green', bold=True)}") click.echo( f" --> To edit the project's config, run {click.style('mallennlp edit', fg='yellow')} " f"from within the project " f"directory or edit the {click.style('Project.toml', fg='green')} file directly." ) click.echo( f" --> To serve the dashboard, run {click.style('mallennlp serve', fg='yellow')} " f"from within the project directory.") click.echo( f" --> You can log in to the dashboard with the username {click.style(username, bold=True, fg='green')}" )
def get_metric_plot_figure(es: ExperimentService, metric_name: str): epochs = es.get_epochs() training_metric_name = f"training_{metric_name}" validation_metric_name = f"validation_{metric_name}" x_vals = list(range(len(epochs))) training_metrics: List[float] = [] validation_metrics: List[float] = [] for epoch in epochs: metrics = epoch.metrics.data if metrics is None: continue t = metrics.get(training_metric_name) if t: training_metrics.append(t) v = metrics.get(validation_metric_name) if v: validation_metrics.append(v) data: List[Dict[str, Any]] = [] if len(training_metrics) == len(epochs): data.append( { "name": training_metric_name, "x": x_vals, "y": training_metrics, "mode": "lines+markers", "hoverinfo": "text", "text": [ f"epoch {i} {training_metric_name}: {m:.4f}" for i, m in enumerate(training_metrics) ], } ) if len(validation_metrics) == len(epochs): data.append( { "name": validation_metric_name, "x": x_vals, "y": validation_metrics, "mode": "lines+markers", "hoverinfo": "text", "text": [ f"epoch {i} {validation_metric_name}: {m:.4f}" for i, m in enumerate(validation_metrics) ], } ) return { "data": data, "layout": { "clickmode": "event+select", "xaxis": { "range": [0, len(epochs) - 0.5], "tickvals": list(range(len(epochs))), "ticktext": [str(i) for i in range(0, len(epochs))], }, "margin": {"l": 40, "b": 30, "t": 20, "pad": 2}, "uirevision": True, }, }
def get_status(es: ExperimentService) -> Status: return es.get_status()