def grant_crowd_qualification(self,
qualification_name: str,
value: int = 1) -> None:
"""
Grant a qualification by the given name to this worker. Check the local
MTurk db to find the matching MTurk qualification to grant, and pass
that. If no qualification exists, try to create one.
In creating a new qualification, Mephisto resolves the ambiguity over which
requester to associate that qualification with by using the FIRST requester
of the given account type (either `mturk` or `mturk_sandbox`)
"""
mturk_qual_details = self.datastore.get_qualification_mapping(
qualification_name)
if mturk_qual_details is not None:
requester = Requester.get(self.db,
mturk_qual_details["requester_id"])
qualification_id = mturk_qual_details["mturk_qualification_id"]
else:
target_type = ("mturk_sandbox" if
qualification_name.endswith("sandbox") else "mturk")
requester = self.db.find_requesters(provider_type=target_type)[-1]
assert isinstance(
requester, MTurkRequester
), "find_requesters must return mturk requester for given provider types"
qualification_id = requester._create_new_mturk_qualification(
qualification_name)
assert isinstance(
requester,
MTurkRequester), "Must be an MTurk requester for MTurk quals"
client = self._get_client(requester._requester_name)
give_worker_qualification(client, self.get_mturk_worker_id(),
qualification_id, value)
return None
def test_requester(self) -> None:
"""Ensure we can create and use a requester"""
db: MephistoDB = self.db
RequesterClass = self.CrowdProviderClass.RequesterClass
test_requester = RequesterClass.new(db, self.get_test_requester_name())
test_requester_2 = Requester.get(db, test_requester.db_id)
self.assertEqual(
test_requester.requester_name,
test_requester_2.requester_name,
"Requester gotten from db not same as first init",
)
# Ensure credential registration works
# TODO(#97) ensure registration fails when we programatically login to an account
# in the future
# self.assertFalse(test_requester.is_registered())
test_requester.register()
self.assertTrue(test_requester.is_registered())
# Ensure requester methods work
avail_budget = test_requester.get_available_budget()
test_budget = self.get_test_requester_balance(test_requester.requester_name)
self.assertEqual(
avail_budget,
test_budget,
"Queried budget from `get_available_budget` not equal to `test_budget`",
)
def test_requester(self) -> None:
"""Test creation and querying of requesters"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
# Check creation and retrieval of a requester
requester_name = "test_requester"
provider_type = PROVIDER_TYPE
requester_id = db.new_requester(requester_name, provider_type)
self.assertIsNotNone(requester_id)
self.assertTrue(isinstance(requester_id, str))
requester_row = db.get_requester(requester_id)
self.assertEqual(requester_row["requester_name"], requester_name)
requester = Requester.get(db, requester_id)
self.assertEqual(requester.requester_name, requester_name)
# Check finding for requesters
requesters = db.find_requesters()
self.assertEqual(len(requesters), 1)
self.assertTrue(isinstance(requesters[0], Requester))
self.assertEqual(requesters[0].db_id, requester_id)
self.assertEqual(requesters[0].requester_name, requester_name)
# Check finding for specific requesters
requesters = db.find_requesters(requester_name=requester_name)
self.assertEqual(len(requesters), 1)
self.assertTrue(isinstance(requesters[0], Requester))
self.assertEqual(requesters[0].db_id, requester_id)
self.assertEqual(requesters[0].requester_name, requester_name)
requesters = db.find_requesters(requester_name="fake_name")
self.assertEqual(len(requesters), 0)
def get_requester(self) -> Requester:
"""
Return the requester that started this task.
"""
if self.__requester is None:
self.__requester = Requester.get(self.db, self.requester_id)
return self.__requester
def get_requester(self) -> Requester:
"""
Return the requester who offered this Assignment
"""
if self.__requester is None:
if self.__task_run is not None:
self.__requester = self.__task_run.get_requester()
else:
self.__requester = Requester.get(self.db, self.requester_id)
return self.__requester
def cleanup_qualification(self, qualification_name: str) -> None:
"""Remove the qualification from the sandbox server, if it exists"""
mapping = self.datastore.get_qualification_mapping(qualification_name)
if mapping is None:
return None
requester_id = mapping["requester_id"]
requester = Requester.get(self.db, requester_id)
assert isinstance(requester,
MTurkRequester), "Must be an mturk requester"
client = requester._get_client(requester._requester_name)
delete_qualification(client, mapping["mturk_qualification_id"])
def direct_soft_block_mturk_workers(
db: "MephistoDB",
worker_list: List[str],
soft_block_qual_name: str,
requester_name: Optional[str] = None,
):
"""
Directly assign the soft blocking MTurk qualification that Mephisto
associates with soft_block_qual_name to all of the MTurk worker ids
in worker_list. If requester_name is not provided, it will use the
most recently registered mturk requester in the database.
"""
reqs = db.find_requesters(requester_name=requester_name,
provider_type="mturk")
requester = reqs[-1]
assert isinstance(
requester, MTurkRequester
), "Can only direct soft block mturk workers from mturk requester"
mturk_qual_details = requester.datastore.get_qualification_mapping(
soft_block_qual_name)
if mturk_qual_details is not None:
# Overrule the requester, as this qualification already exists
requester = Requester.get(db, mturk_qual_details["requester_id"])
qualification_id = mturk_qual_details["mturk_qualification_id"]
else:
qualification_id = requester._create_new_mturk_qualification(
soft_block_qual_name)
assert isinstance(
requester, MTurkRequester
), "Can only direct soft block mturk workers from mturk requester"
mturk_client = requester._get_client(requester._requester_name)
for worker_id in tqdm(worker_list):
try:
give_worker_qualification(mturk_client,
worker_id,
qualification_id,
value=1)
except Exception as e:
print(
f'Failed to give worker with ID: "{worker_id}" qualification with error: {e}. Skipping.'
)
def revoke_crowd_qualification(self, qualification_name: str) -> None:
"""
Revoke the qualification by the given name from this worker. Check the local
MTurk db to find the matching MTurk qualification to revoke, pass if
no such qualification exists.
"""
mturk_qual_details = self.datastore.get_qualification_mapping(
qualification_name)
if mturk_qual_details is None:
logger.error(
f"No locally stored MTurk qualification to revoke for name {qualification_name}"
)
return None
requester = Requester.get(self.db, mturk_qual_details["requester_id"])
assert isinstance(
requester,
MTurkRequester), "Must be an MTurk requester from MTurk quals"
client = self._get_client(requester._requester_name)
qualification_id = mturk_qual_details["mturk_qualification_id"]
remove_worker_qualification(client, self.get_mturk_worker_id(),
qualification_id)
return None
def test_requester_fails(self) -> None:
"""Ensure requesters fail to be created or loaded under failure conditions"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
# Cant get non-existent entry
with self.assertRaises(EntryDoesNotExistException):
requester = Requester.get(db, self.get_fake_id("Requester"))
requester_name = "test_requester"
provider_type = PROVIDER_TYPE
requester_id = db.new_requester(requester_name, provider_type)
# Can't create same requester again
with self.assertRaises(EntryAlreadyExistsException):
requester_id = db.new_requester(requester_name, provider_type)
# Can't use no name
with self.assertRaises(MephistoDBException):
requester_id = db.new_requester("", provider_type)
# Ensure no requesters were created
requesters = db.find_requesters()
self.assertEqual(len(requesters), 1)