def test_mitl_in_internal_config(self):
""" Test that mitl config can be part of the internal configuration """
model = a_manifest().to_primitive()
mitl_config = {
"n_gt": 200,
"min_tasks_in_job": 1000,
"n_gt_sample_min": 1,
"n_gt_sample_max": 3,
"task_max_repeats": 25,
"max_tasks_in_job": 36000,
"model_id": "ResNext50_32x4d",
"task_selection_id": "MinMargin",
"requester_min_repeats": 12,
"requester_max_repeats": 25,
"stop_n_active": 1000,
"requester_accuracy_target": 0.8,
"nested_config": {
"value_a": 1,
"value_b": 2
}
}
model["internal_config"]["mitl"] = mitl_config
manifest = basemodels.Manifest(model)
manifest.validate()
self.assertTrue(True)
def test_gets_default_restrictedanswerset(self):
"""Make sure that the image_label_area_select jobs get a default RAS"""
model = {
'job_mode': 'batch',
'request_type': 'image_label_area_select',
'unsafe_content': False,
'task_bid_price': 1,
'oracle_stake': 0.1,
'expiration_date': 0,
'minimum_trust_server': .1,
'minimum_trust_client': .1,
'requester_accuracy_target': .1,
'recording_oracle_addr': REC_ORACLE,
'reputation_oracle_addr': REP_ORACLE,
'reputation_agent_addr': REP_ORACLE,
'instant_result_delivery_webhook': CALLBACK_URL,
'requester_question': {
"en": "How much money are we to make"
},
'requester_question_example': FAKE_URL,
'job_total_tasks': 5,
'taskdata_uri': FAKE_URL
}
manifest = basemodels.Manifest(model)
manifest.validate()
self.assertGreater(len(manifest['requester_restricted_answer_set'].keys()), 0)
def a_manifest(number_of_tasks=100,
bid_amount=1.0,
oracle_stake=0.05,
expiration_date=0,
minimum_trust=.1,
request_type=IMAGE_LABEL_BINARY,
request_config=None,
job_mode='batch',
multi_challenge_manifests=None) -> basemodels.Manifest:
internal_config = {'exchange': {'a': 1, 'b': 'c'}}
model = {
'requester_restricted_answer_set': {
'0': {
'en': 'English Answer 1'
},
'1': {
'en': 'English Answer 2',
'answer_example_uri': 'https://hcaptcha.com/example_answer2.jpg'
}
},
'job_mode': job_mode,
'request_type': request_type,
'internal_config': internal_config,
'multi_challenge_manifests': multi_challenge_manifests,
'unsafe_content': False,
'task_bid_price': bid_amount,
'oracle_stake': oracle_stake,
'expiration_date': expiration_date,
'minimum_trust_server': minimum_trust,
'minimum_trust_client': minimum_trust,
'requester_accuracy_target': minimum_trust,
'recording_oracle_addr': REC_ORACLE,
'reputation_oracle_addr': REP_ORACLE,
'reputation_agent_addr': REP_ORACLE,
'instant_result_delivery_webhook': CALLBACK_URL,
'requester_question': {
"en": "How much money are we to make"
},
'requester_question_example': FAKE_URL,
'job_total_tasks': number_of_tasks,
'taskdata_uri': FAKE_URL
}
if request_config:
model.update({'request_config': request_config})
manifest = basemodels.Manifest(model)
manifest.validate()
return manifest
def test_manifest(
number_of_tasks=100,
bid_amount=1.0,
oracle_stake=0.05,
expiration_date=0,
minimum_trust=0.1,
request_type=IMAGE_LABEL_BINARY,
request_config=None,
job_mode="batch",
) -> basemodels.Manifest:
model = {
"requester_restricted_answer_set": {
"0": {
"en": "English Answer 1"
},
"1": {
"en": "English Answer 2",
"answer_example_uri":
"https://hcaptcha.com/example_answer2.jpg",
},
},
"job_mode": job_mode,
"request_type": request_type,
"unsafe_content": False,
"task_bid_price": bid_amount,
"oracle_stake": oracle_stake,
"expiration_date": expiration_date,
"minimum_trust_server": minimum_trust,
"minimum_trust_client": minimum_trust,
"requester_accuracy_target": minimum_trust,
"recording_oracle_addr": GAS_PAYER,
"reputation_oracle_addr": GAS_PAYER,
"reputation_agent_addr": GAS_PAYER,
"instant_result_delivery_webhook": CALLBACK_URL,
"requester_question": {
"en": "How much money are we to make"
},
"requester_question_example": FAKE_URL,
"job_total_tasks": number_of_tasks,
"taskdata_uri": FAKE_URL,
}
if request_config:
model.update({"request_config": request_config})
manifest = basemodels.Manifest(model)
manifest.validate()
return manifest
def create_manifest(data: dict):
if test_mode == SCHEMATICS:
return basemodels.Manifest(data)
return pydantic_basemodels.Manifest.construct(**data)
def test_realistic_multi_challenge_example(self):
""" validates a realistic multi_challenge manifest """
obj = {
'job_mode': 'batch',
'request_type': 'image_label_area_select',
'unsafe_content': False,
'task_bid_price': 1,
'oracle_stake': 0.1,
'expiration_date': 0,
'minimum_trust_server': .1,
'minimum_trust_client': .1,
'requester_accuracy_target': .1,
'job_total_tasks': 1000,
'recording_oracle_addr': REC_ORACLE,
'reputation_oracle_addr': REP_ORACLE,
'reputation_agent_addr': REP_ORACLE,
"job_id": "c26c2e6a-41ab-4218-b39e-6314b760c45c",
"request_type": "multi_challenge",
"requester_question": {
"en": "Please draw a bow around the text shown, select the best corresponding labels, and enter the word depicted by the image."
},
"multi_challenge_manifests": [{
"request_type": "image_label_area_select",
"job_id": "c26c2e6a-41ab-4218-b39e-6314b760c45c",
"requester_question": {
"en": "Please draw a bow around the text shown."
},
"request_config": {
"shape_type": "polygon",
"min_points": 1,
"max_points": 4,
"min_shapes_per_image": 1,
"max_shapes_per_image": 4
}
},
{
"request_type": "image_label_multiple_choice",
"job_id": "c26c2e6a-41ab-4218-b39e-6314b760c45c",
"requester_question": {
"en": "Select the corresponding label."
},
"requester_restricted_answer_set": {
"print": {
"en": "Print"
},
"hand-writing": {
"en": "Hand Writing"
}
},
"request_config": {
"multiple_choice_max_choices": 1
}
},
{
"request_type": "image_label_multiple_choice",
"job_id": "c26c2e6a-41ab-4218-b39e-6314b760c45c",
"requester_question": {
"en": "Select the corresponding labels."
},
"requester_restricted_answer_set": {
"top-bottom": {
"en": "Top to Bottom"
},
"bottom-top": {
"en": "Bottom to Top"
},
"left-right": {
"en": "Left to Right"
},
"right-left": {
"en": "Right to Left"
}
},
"request_config": {
"multiple_choice_max_choices": 1
}
},
{
"request_type": "image_label_text",
"job_id": "c26c2e6a-41ab-4218-b39e-6314b760c45c",
"requester_question": {
"en": "Please enter the word in the image."
}
}],
"taskdata": [{
"datapoint_hash": "sha1:5daf66c6031df7f8913bfa0b52e53e3bcd42aab3",
"datapoint_uri": "http://test.com/task.jpg",
"task_key": "2279daef-d10a-4b0f-85d1-0ccbf7c8906b"
}]
}
model = basemodels.Manifest(obj)
# print(model.to_primitive())
self.assertTrue(model.validate() is None)