def mephistoDBReader(): from mephisto.abstractions.databases.local_database import LocalMephistoDB from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser db = LocalMephistoDB() mephisto_data_browser = MephistoDataBrowser(db=db) units = mephisto_data_browser.get_units_for_task_name(database_task_name) for unit in units: yield mephisto_data_browser.get_data_from_unit(unit)
def __init__(self, opt: Dict, remove_failed: bool = True): """ Initialize the analyzer. Builds up the dataframe :param opt: opt dict :param remove_failed: Whether to remove ratings from turkers who failed onboarding """ assert ',' not in opt['run_ids'], "AcuteAnalyzer can only handle one run ID!" self.run_id = opt['run_ids'] self.pairings_filepath = opt['pairings_filepath'] self.outdir = opt['outdir'] self.root_dir = opt['root_dir'] # Get task for loading pairing files self.task = opt.get('task', 'q') if opt.get('model_ordering') is not None: self.custom_model_ordering = opt['model_ordering'].split(',') else: self.custom_model_ordering = None if not self.outdir or not self.pairings_filepath: # Default to using self.root_dir as the root directory for outputs assert self.root_dir is not None and os.path.isdir( self.root_dir ), '--root-dir must be a real directory!' if not self.pairings_filepath: # Will be set to a non-empty path later self.pairings_filepath = '' if not self.outdir: self.outdir = os.path.join(self.root_dir, f'{self.run_id}-results') if not os.path.exists(self.outdir): os.makedirs(self.outdir, exist_ok=True) mephisto_root_path = opt['mephisto_root'] if not mephisto_root_path: mephisto_root_path = None self.mephisto_db = LocalMephistoDB(database_path=mephisto_root_path) self.mephisto_data_browser = MephistoDataBrowser(db=self.mephisto_db) self.checkbox_prefix = self.CHECKBOX_PREFIX # Prepended to checkbox columns in self.dataframe self.dataframe = self._extract_to_dataframe() self._check_eval_question() if remove_failed: self._remove_failed_onboarding() if self.dataframe.index.size == 0: raise ValueError('No valid results found!') self._get_model_nick_names() self._load_pairing_files()
def mephistoDBReader(): from mephisto.abstractions.databases.local_database import LocalMephistoDB from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser db = LocalMephistoDB() mephisto_data_browser = MephistoDataBrowser(db=db) def format_data_for_review(data): contents = data["data"] return f"{data}" units = mephisto_data_browser.get_units_for_task_name( database_task_name) for unit in units: yield format_data_for_review( mephisto_data_browser.get_data_from_unit(unit))
def review( review_app_dir, port, output, output_method, csv_headers, json, database_task_name, all_data, debug, ): """Launch a local review UI server. Reads in rows froms stdin and outputs to either a file or stdout.""" from mephisto.client.review.review_server import run if output == "" and output_method == "file": raise click.UsageError( "You must specify an output file via --output=<filename>, unless the --stdout flag is set." ) if database_task_name is not None: from mephisto.abstractions.databases.local_database import LocalMephistoDB from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser db = LocalMephistoDB() mephisto_data_browser = MephistoDataBrowser(db=db) name_list = mephisto_data_browser.get_task_name_list() if database_task_name not in name_list: raise click.BadParameter( f'The task name "{database_task_name}" did not exist in MephistoDB.\n\nPerhaps you meant one of these? {", ".join(name_list)}\n\nFlag usage: mephisto review --db [task_name]\n' ) run( review_app_dir, port, output, csv_headers, json, database_task_name, all_data, debug, )
def get_mephisto_data_browser(self) -> MephistoDataBrowser: if not self._mephisto_data_browser: db = self.get_mephisto_db() self._mephisto_data_browser = MephistoDataBrowser(db=db) return self._mephisto_data_browser
assignment = Assignment(db, assignment_id) assignment.write_assignment_data( InitializationData(unit_data={}, shared=annotation["inputs"]) ) unit_id = db.new_unit( task_run.task_id, task_run.db_id, task_run.requester_id, assignment_id, 0, # Unit_index 0, # reward task_run.provider_type, task_run.task_type, task_run.sandbox, ) unit = Unit(db, unit_id) agent = MockAgent.new(db, worker, unit) agent.state.state["inputs"] = annotation["inputs"] agent.state.state["outputs"] = annotation["outputs"] agent.state.save_data() agent.mark_done() agent.update_status(AgentState.STATUS_COMPLETED) # Show tasks appear in MephistoDB: mephisto_data_browser = MephistoDataBrowser(db=db) units = mephisto_data_browser.get_units_for_task_name(input("Input task name: ")) for unit in units: print(mephisto_data_browser.get_data_from_unit(unit))