class TaskDataEntry(Model):
"""
Taskdata file format:
[
{
"task_key": "407fdd93-687a-46bb-b578-89eb96b4109d",
"datapoint_uri": "https://domain.com/file1.jpg",
"datapoint_hash": "f4acbe8562907183a484498ba901bfe5c5503aaa"
},
{
"task_key": "20bd4f3e-4518-4602-b67a-1d8dfabcce0c",
"datapoint_uri": "https://domain.com/file2.jpg",
"datapoint_hash": "f4acbe8562907183a484498ba901bfe5c5503aaa"
}
]
"""
task_key = UUIDType()
datapoint_uri = URLType(required=True, min_length=10)
datapoint_hash = StringType()
metadata = DictType(UnionType(
[StringType(required=False, max_length=256), FloatType, IntType]),
required=False)
def validate_metadata(self, data, value):
if len(str(value)) > 1024:
raise ValidationError("metadata should be < 1024")
return value
class NetworkmetadataModel(MetadataModel):
"""Model that describes the expected structure of the network metadata"""
computational_metadata = DictType(UnionType(types=(StringType, FloatType)),
required=True)
directionality = StringType(regex=r"(undirected|directed)", required=True)
interaction_type = StringType()
def test_regular_union_type(self):
"""
Test that shows tricky behaviour of regular union type
"""
union_def = UnionType((IntType, FloatType, StringType, BooleanType))
self.assertEqual(
union_def('464'), 464
) # Notice the conversion! It depends on order of types in the construct.
self.assertEqual(union_def(464), 464)
class Preprocess(Model):
pipeline = StringType(
required=True, choices=["FaceBlurPipeline", "OCRThinFilterPipeline"])
config = DictType(UnionType([FloatType, IntType, StringType]))
def to_dict(self):
p = {"pipeline": self.pipeline}
if self.config is not None:
p["config"] = self.config
return p
class Product(Model):
service_code = StringType()
region_name = StringType()
product_family = StringType()
sku = StringType()
attributes = DictType(UnionType([IntType(), StringType(), FloatType()]))
publication_date = DateTimeType(serialize_when_none=False)
version = StringType()
terms = ListType(ModelType(Terms), default=[])
def reference(self):
return {
"resource_id": self.sku
}
class Transaction(Model):
hash = StringType(required=True)
type = StringType(required=True)
body = UnionType(
(
ModelType(CreateAccountTransaction),
ModelType(DeployContractTransaction),
ModelType(FunctionCallTransaction),
ModelType(SendMoneyTransaction),
ModelType(StakeTransaction),
ModelType(SwapKeyTransaction),
),
required=True,
)
receipts = ListType(ModelType(Receipt), required=True, default=[])
class InternalConfig(Model):
""" discarded from incoming manifests """
exchange = DictType(StringType, UnionType([StringType, IntType,
FloatType]))
reco = DictType(StringType, UnionType([StringType, IntType, FloatType]))
repo = DictType(StringType, UnionType([StringType, IntType, FloatType]))
other = DictType(StringType, UnionType([StringType, IntType, FloatType]))
# Accept one layer of nested
mitl = DictType(
UnionType([
StringType,
IntType,
FloatType,
DictType(UnionType([StringType, IntType, FloatType])),
]))
class MetricDataModel(Model):
labels = ListType(DictType(IntType), required=True)
values = ListType(UnionType((FloatType, IntType)), required=True)
class Manifest(Model):
""" The manifest description. """
job_mode = StringType(required=True,
choices=["batch", "online", "instant_delivery"])
job_api_key = UUIDType(default=uuid.uuid4)
job_id = UUIDType(default=uuid.uuid4)
job_total_tasks = IntType(required=True)
requester_restricted_answer_set = DictType(DictType(StringType))
def validate_requester_restricted_answer_set(self, data, value):
"""image_label_area_select should always have a single RAS set"""
# validation runs before other params, so need to handle missing case
if not data.get('request_type'):
raise ValidationError("request_type missing")
if data['request_type'] == 'image_label_area_select':
if not value or len(value.keys()) == 0:
value = {'label': {}}
data['requester_restricted_answer_set'] = value
return value
requester_description = StringType()
requester_max_repeats = IntType(default=100)
requester_min_repeats = IntType(default=1)
requester_question = DictType(StringType)
requester_question_example = UnionType((URLType, ListType), field=URLType)
def validate_requester_question_example(self, data, value):
# validation runs before other params, so need to handle missing case
if not data.get('request_type'):
raise ValidationError("request_type missing")
# based on https://github.com/hCaptcha/hmt-basemodels/issues/27#issuecomment-590706643
supports_lists = ['image_label_area_select', 'image_label_binary']
if isinstance(value,
list) and not data['request_type'] in supports_lists:
raise ValidationError(
"Lists are not allowed in this challenge type")
return value
unsafe_content = BooleanType(default=False)
task_bid_price = DecimalType(required=True)
oracle_stake = DecimalType(required=True)
expiration_date = IntType()
requester_accuracy_target = FloatType(default=.1)
manifest_smart_bounty_addr = StringType()
hmtoken_addr = StringType()
minimum_trust_server = FloatType(default=.1)
minimum_trust_client = FloatType(default=.1)
recording_oracle_addr = StringType(required=True)
reputation_oracle_addr = StringType(required=True)
reputation_agent_addr = StringType(required=True)
requester_pgp_public_key = StringType()
ro_uri = StringType()
repo_uri = StringType()
batch_result_delivery_webhook = URLType()
online_result_delivery_webhook = URLType()
instant_result_delivery_webhook = URLType()
multi_challenge_manifests = ListType(ModelType(NestedManifest),
required=False)
request_type = StringType(required=True,
choices=BASE_JOB_TYPES + ["multi_challenge"])
validate_request_type = validate_request_type
request_config = ModelType(RequestConfig, required=False)
# If taskdata is directly provided
taskdata = ListType(ModelType(TaskData))
# If taskdata is separately stored
taskdata_uri = URLType()
# Groundtruth data is stored as a URL or optionally as an inlined json-serialized stringtype
groundtruth_uri = URLType(required=False)
groundtruth = StringType(required=False)
rejected_uri = URLType(required=False)
rejected_count = IntType(default=0, required=False)
def validate_groundtruth(self, data, value):
if data.get('groundtruth_uri') and data.get('groundtruth'):
raise ValidationError(
"Specify only groundtruth_uri or groundtruth, not both.")
return value
# internal config options for param tests etc.
internal_config = ModelType(InternalConfig, required=False)
# Configuration id -- XXX LEGACY
confcalc_configuration_id = StringType(required=False)
restricted_audience = ModelType(RestrictedAudience,
required=True,
default={})
def validate_taskdata_uri(self, data, value):
if data.get('taskdata') and len(
data.get('taskdata')) > 0 and data.get('taskdata_uri'):
raise ValidationError(
u'Specify only one of taskdata {} or taskdata_uri {}'.format(
data.get('taskdata'), data.get('taskdata_uri')))
return value
validate_taskdata = validate_taskdata_uri
webhook = ModelType(Webhook)
class NestedManifest(Model):
""" The nested manifest description for multi_challenge jobs """
job_id = UUIDType(default=uuid.uuid4)
requester_restricted_answer_set = DictType(DictType(StringType))
def validate_requester_restricted_answer_set(self, data, value):
"""image_label_area_select should always have a single RAS set"""
# validation runs before other params, so need to handle missing case
if not data.get('request_type'):
raise ValidationError("request_type missing")
if data['request_type'] == 'image_label_area_select':
if not value or len(value.keys()) == 0:
value = {'label': {}}
data['requester_restricted_answer_set'] = value
return value
requester_description = StringType()
requester_max_repeats = IntType(default=100)
requester_min_repeats = IntType(default=1)
requester_question = DictType(StringType)
requester_question_example = UnionType((URLType, ListType), field=URLType)
def validate_requester_question_example(self, data, value):
# validation runs before other params, so need to handle missing case
if not data.get('request_type'):
raise ValidationError("request_type missing")
# based on https://github.com/hCaptcha/hmt-basemodels/issues/27#issuecomment-590706643
supports_lists = ['image_label_area_select', 'image_label_binary']
if isinstance(value,
list) and not data['request_type'] in supports_lists:
raise ValidationError(
"Lists are not allowed in this challenge type")
return value
unsafe_content = BooleanType(default=False)
requester_accuracy_target = FloatType(default=.1)
request_type = StringType(required=True, choices=BASE_JOB_TYPES)
validate_request_type = validate_request_type
request_config = ModelType(RequestConfig, required=False)
# Groundtruth data is stored as a URL or optionally as an inlined json-serialized stringtype
groundtruth_uri = URLType(required=False)
groundtruth = StringType(required=False)
def validate_groundtruth(self, data, value):
if data.get('groundtruth_uri') and data.get('groundtruth'):
raise ValidationError(
"Specify only groundtruth_uri or groundtruth, not both.")
return value
# Configuration id -- XXX LEGACY
confcalc_configuration_id = StringType(required=False)
webhook = ModelType(Webhook)
class Foo(XMLModel):
union = UnionType([IntType, StringType])
class ILASGroundtruthEntry(Model):
entity_name = UnionType([IntType, FloatType])
entity_type = StringType()
entity_coords = ListType(UnionType([IntType, FloatType]))
