def mephistoDBReader():
from mephisto.abstractions.databases.local_database import LocalMephistoDB
from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser
db = LocalMephistoDB()
mephisto_data_browser = MephistoDataBrowser(db=db)
units = mephisto_data_browser.get_units_for_task_name(database_task_name)
for unit in units:
yield mephisto_data_browser.get_data_from_unit(unit)
def __init__(self, opt: Dict, remove_failed: bool = True):
"""
Initialize the analyzer.
Builds up the dataframe
:param opt:
opt dict
:param remove_failed:
Whether to remove ratings from turkers who failed onboarding
"""
assert ',' not in opt['run_ids'], "AcuteAnalyzer can only handle one run ID!"
self.run_id = opt['run_ids']
self.pairings_filepath = opt['pairings_filepath']
self.outdir = opt['outdir']
self.root_dir = opt['root_dir']
# Get task for loading pairing files
self.task = opt.get('task', 'q')
if opt.get('model_ordering') is not None:
self.custom_model_ordering = opt['model_ordering'].split(',')
else:
self.custom_model_ordering = None
if not self.outdir or not self.pairings_filepath:
# Default to using self.root_dir as the root directory for outputs
assert self.root_dir is not None and os.path.isdir(
self.root_dir
), '--root-dir must be a real directory!'
if not self.pairings_filepath:
# Will be set to a non-empty path later
self.pairings_filepath = ''
if not self.outdir:
self.outdir = os.path.join(self.root_dir, f'{self.run_id}-results')
if not os.path.exists(self.outdir):
os.makedirs(self.outdir, exist_ok=True)
mephisto_root_path = opt['mephisto_root']
if not mephisto_root_path:
mephisto_root_path = None
self.mephisto_db = LocalMephistoDB(database_path=mephisto_root_path)
self.mephisto_data_browser = MephistoDataBrowser(db=self.mephisto_db)
self.checkbox_prefix = self.CHECKBOX_PREFIX
# Prepended to checkbox columns in self.dataframe
self.dataframe = self._extract_to_dataframe()
self._check_eval_question()
if remove_failed:
self._remove_failed_onboarding()
if self.dataframe.index.size == 0:
raise ValueError('No valid results found!')
self._get_model_nick_names()
self._load_pairing_files()
def mephistoDBReader():
from mephisto.abstractions.databases.local_database import LocalMephistoDB
from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser
db = LocalMephistoDB()
mephisto_data_browser = MephistoDataBrowser(db=db)
def format_data_for_review(data):
contents = data["data"]
return f"{data}"
units = mephisto_data_browser.get_units_for_task_name(
database_task_name)
for unit in units:
yield format_data_for_review(
mephisto_data_browser.get_data_from_unit(unit))
def review(
review_app_dir,
port,
output,
output_method,
csv_headers,
json,
database_task_name,
all_data,
debug,
):
"""Launch a local review UI server. Reads in rows froms stdin and outputs to either a file or stdout."""
from mephisto.client.review.review_server import run
if output == "" and output_method == "file":
raise click.UsageError(
"You must specify an output file via --output=<filename>, unless the --stdout flag is set."
)
if database_task_name is not None:
from mephisto.abstractions.databases.local_database import LocalMephistoDB
from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser
db = LocalMephistoDB()
mephisto_data_browser = MephistoDataBrowser(db=db)
name_list = mephisto_data_browser.get_task_name_list()
if database_task_name not in name_list:
raise click.BadParameter(
f'The task name "{database_task_name}" did not exist in MephistoDB.\n\nPerhaps you meant one of these? {", ".join(name_list)}\n\nFlag usage: mephisto review --db [task_name]\n'
)
run(
review_app_dir,
port,
output,
csv_headers,
json,
database_task_name,
all_data,
debug,
)
def get_mephisto_data_browser(self) -> MephistoDataBrowser:
if not self._mephisto_data_browser:
db = self.get_mephisto_db()
self._mephisto_data_browser = MephistoDataBrowser(db=db)
return self._mephisto_data_browser
assignment = Assignment(db, assignment_id)
assignment.write_assignment_data(
InitializationData(unit_data={}, shared=annotation["inputs"])
)
unit_id = db.new_unit(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
assignment_id,
0, # Unit_index
0, # reward
task_run.provider_type,
task_run.task_type,
task_run.sandbox,
)
unit = Unit(db, unit_id)
agent = MockAgent.new(db, worker, unit)
agent.state.state["inputs"] = annotation["inputs"]
agent.state.state["outputs"] = annotation["outputs"]
agent.state.save_data()
agent.mark_done()
agent.update_status(AgentState.STATUS_COMPLETED)
# Show tasks appear in MephistoDB:
mephisto_data_browser = MephistoDataBrowser(db=db)
units = mephisto_data_browser.get_units_for_task_name(input("Input task name: "))
for unit in units:
print(mephisto_data_browser.get_data_from_unit(unit))