def test_to_python__invalid_val():
field = ProbabilityField()
val = 1.1
with pytest.raises(exceptions.ValidationError):
field.to_python(val)
val = -1.0
with pytest.raises(exceptions.ValidationError):
field.to_python(val)
class CriterionRules(models.Model):
threshold = ProbabilityField(
verbose_name="Threshold",
help_text="Minimum value for the answer to be accepted",
)
class Meta:
abstract = True
def __str__(self):
raise NotImplementedError("This method has to be implemented.")
@staticmethod
def get_or_create(*args, **kwargs):
raise NotImplementedError("This method has to be implemented.")
def __iter__(self):
return {
field.name: {
"name":
field.name,
"full_name":
field.verbose_name,
"description":
field.help_text,
"value":
[instance.pk for instance in getattr(self, field.name).all()]
if field.__class__.__name__ == "ManyToManyField" else getattr(
self, field.name),
"type":
field.__class__.__name__,
"allowed":
getattr(self, field.name).model.available()
if field.__class__.__name__ == "ManyToManyField" else getattr(
field, "allowed", None),
}
for field in self.__class__._meta.get_fields()
if field.name != "id" and not field.name.endswith("ptr")
and not field.__class__.__name__ == "ManyToOneRel"
}.iteritems()
class SelectedAnswerCriterionRules(CriterionRules):
default_if_never_shown = ProbabilityField(
verbose_name="Default value if never shown",
help_text="Value to evaluate to if answer never shown before.",
)
def __str__(self):
return "Rules {} for criterion selected_answer".format(self.pk)
@staticmethod
def get_or_create(threshold=1, default_if_never_shown=0):
"""
Creates or get the criterion rules.
Parameters
----------
threshold : float in [0, 1] (default : 0)
Minimum value for the criterion to pass
default_if_never_shown : float in [0, 1] (default : 0)
Value to show if answer never shown before
Returns
-------
SelectedAnswerCriterionRules
Instance
Raises
------
ValueError
If the arguments have invalid values
"""
if threshold < 0 or threshold > 1:
raise ValueError("The threshold must be between 0 and 1")
if default_if_never_shown < 0 or default_if_never_shown > 1:
raise ValueError(
"The default_if_never_shown must be between 0 and 1")
criterion, __ = SelectedAnswerCriterionRules.objects.get_or_create(
threshold=threshold, default_if_never_shown=default_if_never_shown)
return criterion
class LikelihoodCache(models.Model):
answer = models.PositiveIntegerField(null=True, blank=True)
language = models.ForeignKey(LikelihoodLanguage, on_delete=models.CASCADE)
hash = models.CharField(max_length=32, unique=True, db_index=True)
likelihood = ProbabilityField()
likelihood_random = ProbabilityField()
@classmethod
def get(cls, answer, language, max_gram):
if isinstance(answer, str):
answer_pk = None
rationale = answer
else:
answer_pk = answer.pk
rationale = answer.rationale
hash_ = hashlib.md5(
json.dumps(
{
"text": rationale,
"language": language.language,
"max_gram": max_gram,
}
).encode()
).hexdigest()
try:
cache = cls.objects.get(hash=hash_)
likelihood = cache.likelihood
likelihood_random = cache.likelihood_random
except cls.DoesNotExist:
predict = create_model(
language.language,
language.n_gram_urls,
language.left_to_right,
max_gram,
)
likelihood, likelihood_random = predict(rationale)
# Because multiple servers are used, sometimes the likelihood is
# written to the db by the first server while the second one is
# computing it. In these cases, the likelihood written to the db
# will be used.
try:
cls.objects.create(
answer=answer_pk,
language=language,
hash=hash_,
likelihood=likelihood,
likelihood_random=likelihood_random,
)
except IntegrityError:
cache = cls.objects.get(hash=hash_)
likelihood = cache.likelihood
likelihood_random = cache.likelihood_random
return likelihood, likelihood_random
@classmethod
def batch(cls, answers, language, max_gram):
answers = list(answers)
pks = [
None if isinstance(answer, str) else answer.pk
for answer in answers
]
rationales = [
answer if isinstance(answer, str) else answer.rationale
for answer in answers
]
hashes = [
hashlib.md5(
json.dumps(
{
"text": rationale,
"language": language.language,
"max_gram": max_gram,
}
).encode()
).hexdigest()
for rationale in rationales
]
likelihoods = []
for hash_ in hashes:
try:
cache = cls.objects.get(hash=hash_)
likelihoods.append((cache.likelihood, cache.likelihood_random))
except cls.DoesNotExist:
likelihoods.append(None)
if not all(likelihoods):
predict = create_model(
language.language,
language.n_gram_urls,
language.left_to_right,
max_gram,
)
for (i, likelihood), pk, rationale, hash_ in zip(
enumerate(likelihoods), pks, rationales, hashes
):
if likelihood is None:
_likelihood, likelihood_random = predict(rationale)
try:
cls.objects.create(
answer=pk,
language=language,
hash=hash_,
likelihood=_likelihood,
likelihood_random=likelihood_random,
)
except IntegrityError:
pass
likelihoods[i] = (_likelihood, likelihood_random)
return likelihoods
def test_to_python():
field = ProbabilityField()
val = 1.0
assert field.to_python(val) == val