class TaskSerializer_Simplify(serializers.ModelSerializer):
user = DjangoUserSerializer_Simplify(many=False, read_only=True)
project = ProjectSerializer_Simplify(many=False, read_only=True)
# def get_project(self, obj):
# return obj.id + " TODO change into project simple_serializer"
class Meta:
model = Tasks
fields = ('id', 'type', 'user', 'project')
class PassageSerializer(serializers.ModelSerializer):
created_by = DjangoUserSerializer_Simplify(many=False, read_only=True)
source = SourceSerializer_Simplify(many=False)
class Meta:
model = Passages
fields = ('id', 'text', 'type', 'source', 'text_direction',
'external_id', 'is_active', 'created_by', 'created_at',
'updated_at')
def create(self, validated_data):
ownerUser = self.initial_data['created_by']
validated_data['created_by'] = ownerUser
source = get_object_or_404(Sources,
pk=get_value_or_none(
'id', self.initial_data['source']))
active_obj_or_raise_exeption(source)
validated_data['source'] = source
texts_array = validated_data['text'].split("<DELIMITER>")
for text in texts_array:
validated_data['text'] = text
newPassage = Passages.objects.create(**validated_data)
return newPassage
def update(self, instance, validated_data):
validated_data.pop('source')
print(validated_data)
# prevent update asset that used in another asset
if self.is_used_in_a_task(instance) == False:
updated_source = get_object_or_404(
Sources,
pk=get_value_or_none('id', self.initial_data['source']))
instance.source = validated_data.get('source', updated_source)
instance.text = validated_data.get('text', instance.text)
instance.text_direction = validated_data.get('text_direction',
instance.text_direction)
instance.type = validated_data.get('type', instance.type)
instance.is_active = validated_data.get('is_active',
instance.is_active)
instance.external_id = validated_data.get('external_id',
instance.is_active)
instance.save()
return instance
def is_used_in_a_task(self, instance):
children_list = Tasks.objects.all().filter(passage=instance.id)
is_parent = len(children_list) > 0
return is_parent
class SourceSerializer(serializers.ModelSerializer):
created_by = DjangoUserSerializer_Simplify(many=False, read_only=True)
class Meta:
model = Sources
fields = ('id', 'name', 'text', 'is_active', 'created_by',
'created_at', 'updated_at')
def create(self, validated_data):
ownerUser = self.initial_data['created_by']
validated_data['created_by'] = ownerUser
return Sources.objects.create(**validated_data)
class CategorySerializer(serializers.ModelSerializer):
created_by = DjangoUserSerializer_Simplify(many=False, read_only=True)
class Meta:
model = Categories
fields = ('id', 'name', 'description', 'abbreviation', 'tooltip',
'is_default', 'is_active', 'created_by', 'created_at',
'updated_at')
def create(self, validated_data):
ownerUser = self.initial_data['created_by']
validated_data['created_by'] = ownerUser
return Categories.objects.create(**validated_data)
def get_user(self,obj):
return DjangoUserSerializer_Simplify(obj.annotator).data
class TaskInChartSerializer(serializers.ModelSerializer):
created_by = DjangoUserSerializer_Simplify(many=False, read_only=True)
passage = PassageSerializer_Simplify(many=False, read_only=True,allow_null=True)
project = ProjectSerializer_Simplify(many=False, read_only=True,allow_null=False)
user = serializers.SerializerMethodField()
parent = serializers.SerializerMethodField()
children = serializers.SerializerMethodField()
#out_of_date = serializers.BooleanField(source='out_of_date')
#out_of_date = serializers.SerializerMethodField() # Added Sep 30
def get_user(self,obj):
return DjangoUserSerializer_Simplify(obj.annotator).data
def get_parent(self,obj):
if obj.parent_task is not None:
return TaskSerializer_Simplify(obj.parent_task).data
else:
return None
def get_children(self, obj):
children_tasks = Tasks.objects.all().filter(parent_task_id=obj.id)
children_json = []
for cl in children_tasks:
children_json.append(TaskSerializer_Simplify(cl).data)
return children_json
"""
def get_out_of_date(self,obj):
if obj.project.layer.type != Constants.LAYER_TYPES_JSON['ROOT'] and obj.parent_task is not None:
num_of_submitted_review_tasks = \
Tasks.objects.all().filter(parent_task_id=obj.parent_task.id,\
type=Constants.TASK_TYPES_JSON['REVIEW'],status=Constants.TASK_STATUS_JSON['SUBMITTED']).count()
return (num_of_submitted_review_tasks > 0)
else:
return False
"""
class Meta:
model = Tasks
fields = (
'id',
'parent',
'children',
'type',
'status',
'project',
'user',
'passage',
'is_demo',
'manager_comment',
'user_comment',
'out_of_date',
'obsoleted_by',
'parent_obsoleted_by',
'is_active',
'created_by',
'created_at',
'updated_at'
)
def create(self, validated_data):
ownerUser = self.initial_data['created_by']
validated_data['created_by'] = ownerUser
project = get_object_or_404(Projects, pk=self.initial_data['project']['id'])
annotator = get_object_or_404(Users, pk=get_value_or_none('id',get_value_or_none('user',self.initial_data))) # todo: when coarsening layer task - no need of annotator - check which one to set default?
parent = None
if self.initial_data['parent']:
parent = get_object_or_404(Tasks, pk=get_value_or_none('id', self.initial_data['parent']))
active_obj_or_raise_exeption(parent)
newTask = Tasks()
newTask.created_by = ownerUser
newTask.parent_task = parent
newTask.status = Constants.TASK_STATUS_JSON['NOT_STARTED']
newTask.type = validated_data['type']
newTask.is_demo = validated_data['is_demo']
newTask.manager_comment = validated_data['manager_comment']
newTask.user_comment = validated_data.get('user_comment','')
# Omri Abend (Sep 13)
# tasks cannot be created with is_active=True if their parent is not submitted
if (newTask.parent_task and newTask.parent_task.status != Constants.TASK_STATUS_JSON['SUBMITTED']):
newTask.is_active = False
else:
newTask.is_active = validated_data['is_active']
newTask.project = project
newTask.annotator = annotator
if (newTask.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
passage = get_object_or_404(Passages, pk=get_value_or_none('id',get_value_or_none('passage',self.initial_data)))
else:
passage = self.get_passage_by_parent_task(parent)
newTask.passage = passage
newTask.created_at = timezone.now()
active_obj_or_raise_exeption(project)
active_obj_or_raise_exeption(annotator)
active_obj_or_raise_exeption(passage)
if(newTask.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
newTask.save()
self.generate_and_save_tokens(newTask)
elif(newTask.type == Constants.TASK_TYPES_JSON['ANNOTATION'] or (newTask.type == Constants.TASK_TYPES_JSON['REVIEW'])):
if(self.has_parent_task(newTask) and self.parent_task_layer_is_my_parent_layer(newTask)): # and self.is_parent_task_submitted(newTask)):
self.save_task_by_layer_type(newTask)
return newTask
def update(self, instance, validated_data):
# avoid changing the layer's type
validated_data['type'] = instance.type
# Omri Abend (Sep 13)
# tasks cannot be updated to is_active=True if their parent is not submitted
if (validated_data['is_active'] and instance.parent_task and instance.parent_task.status != Constants.TASK_STATUS_JSON['SUBMITTED']):
raise OnlyActiveIfParentIsActiveException
# allow update only status is_demo is_active manager_comment
# continue updating the layer's attributes
return super(self.__class__, self).update(instance, validated_data)
def get_passage_by_parent_task(self,parent_task):
return parent_task.passage
def is_parent_task_submitted(self,task):
if task.parent_task.status == Constants.TASK_STATUS_JSON['SUBMITTED']:
return True
else:
raise CreateDerivedAnnotationTaskDeniedException
def parent_task_layer_is_my_parent_layer(self,task):
if task.project.layer.parent_layer_id != None: # if im using a derived layer in my new task
parent_task_layer_id = task.parent_task.project.layer.id
my_parent_layer_id = task.project.layer.parent_layer_id.id
print("my_parent_layer_id: "+str(my_parent_layer_id)+" ; parent_task_layer_id: "+str(parent_task_layer_id))
if my_parent_layer_id == parent_task_layer_id:
return True
else:
raise CreateAnnotationTaskDeniedException
else:
return True
def has_parent_task(self,task):
if hasattr(task,'parent_task') and task.parent_task != None:
return True
else:
raise CreateAnnotationTaskDeniedException
def is_parent_of_other_tasks(self, instance):
children_list = Tasks.objects.all().filter(parent_task_id=instance.id)
is_parent = len(children_list) > 0
return is_parent
def generate_and_save_tokens(self,taskInstance):
tokens_arr = tokenize(taskInstance.passage.text.replace('\\n','\n'))
taskInstance.tokens_set.all().delete()
# self.get_object()
for token in tokens_arr:
newToken = Tokens()
newToken.require_annotation = (not token['is_punctuation'])
newToken.text = token['text']
newToken.start_index = token['start_index']
newToken.end_index = token['end_index']
taskInstance.tokens_set.add(newToken,bulk=False)
def save_task_by_layer_type(self,task):
task_layer = task.project.layer
if(task_layer.type == Constants.LAYER_TYPES_JSON['ROOT']):
print('save_task_by_layer_type - ROOT - start')
task.save()
print('save_task_by_layer_type - ROOT - end')
elif (task_layer.type == Constants.LAYER_TYPES_JSON['EXTENSION']):
print('save_task_by_layer_type - EXTENSION - start')
task.save()
print('save_task_by_layer_type - EXTENSION - end')
elif (task_layer.type == Constants.LAYER_TYPES_JSON['REFINEMENT']):
print('save_task_by_layer_type - REFINEMENT - start')
task.save()
print('save_task_by_layer_type - REFINEMENT - end')
elif (task_layer.type == Constants.LAYER_TYPES_JSON['COARSENING']):
self.save_coarsening_task_process(task)
return task
def save_coarsening_task_process(self,task):
print('save_task_by_layer_type - COARSENING - start')
# make sure that the parent task is not tokenization task
if (task.parent_task.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
raise CreateCoarseningAnnotationTaskDeniedException
else:
# set the task type to ONGOING
task.status = Constants.TASK_STATUS_JSON['ONGOING']
# save the task
task.save()
# e.g. there are 3 categories that bacome 1 category so:
parent_task_annotation_units = task.parent_task.annotation_units_set.all().order_by('id')
coarsenned_categories = self.get_coarsenned_categories(task.project.layer)
remote_units_array = []
# go over all of the annotation units in this task
# - for each unit, if one of the old categories exists - replace it with the new category
for parent_au in parent_task_annotation_units:
parent_au = Annotation_UnitsSerializer(parent_au).data
annotation_unit = Annotation_Units()
annotation_unit.tree_id = parent_au['tree_id']
annotation_unit.task_id = Tasks.objects.get(id=parent_au['task_id'])
annotation_unit.type = parent_au['type']
annotation_unit.comment = parent_au['comment']
annotation_unit.gui_status = parent_au['gui_status']
annotation_unit.is_remote_copy = parent_au['is_remote_copy']
parent_annotation_unit = self.get_parent_annotation_unit_or_none(parent_au['parent_id'],task.id)
annotation_unit.parent_id = parent_annotation_unit
task.annotation_units_set.add(annotation_unit, bulk=False)
self.save_coarsening_annotation_categories(annotation_unit, parent_au, coarsenned_categories)
# check if i have a remote units
remote_units = Annotation_Remote_Units_Annotation_Units.objects.all().filter(unit_id=parent_au['id'])
for ru in remote_units:
# retrieve its original unit
remote_original_unit = Annotation_Units.objects.get(id=ru.remote_unit_id.id, task_id=task.parent_task.id)
remote_original_unit_in_coarsening_task = Annotation_Units.objects.get(
tree_id=remote_original_unit.tree_id, task_id=task.id)
unit_id_remote_original_unit = Annotation_Units.objects.get(
id=ru.unit_id.id, task_id=task.parent_task.id)
unit_id_remote_original_unit_in_coarsening_task = Annotation_Units.objects.get(
tree_id=unit_id_remote_original_unit.tree_id, task_id=task.id)
# set the remote is_remote_copy = true
remote_original_unit_in_coarsening_task.is_remote_copy = True
# set the parent_id to be the remote's one
remote_original_unit_in_coarsening_task.parent_id = unit_id_remote_original_unit_in_coarsening_task
# get its original categories
originial_remote_categories = Annotation_Units_Categories.objects.all().filter(
unit_id=remote_original_unit,remote_parent_id=unit_id_remote_original_unit)
# set the remote categories to the coarsening task
remote_categories = []
for cat in originial_remote_categories:
annotation_unit_categories = Annotation_Units_Categories()
annotation_unit_categories.unit_id = remote_original_unit_in_coarsening_task
annotation_unit_categories.remote_parent_id = unit_id_remote_original_unit_in_coarsening_task
annotation_unit_categories.category_id = cat.category_id
annotation_unit_categories.save()
remote_categories.append(annotation_unit_categories)
# TODO: save coaesened category to remote units in coarsening task
if cat.remote_parent_id != None:
# if the category is the coarsened one, add it to the annotation unit
coarsend_category = self.get_coarsening_layer_category_or_none(coarsenned_categories, {'id':cat.category_id.id})
if coarsend_category is not None:
coarsend_unit_category = Annotation_Units_Categories()
coarsend_unit_category.unit_id = remote_original_unit_in_coarsening_task
coarsend_unit_category.category_id = coarsend_category
coarsend_unit_category.remote_parent_id = unit_id_remote_original_unit_in_coarsening_task
try:
coarsend_unit_category.save()
except:
print("already saved")
remote_original_unit_in_coarsening_task.remote_categories = remote_categories
# add the remote original unit to the json output
remote_units_array.append(remote_original_unit_in_coarsening_task)
for annotation_remote_unit in remote_units_array:
remote_unit = self.save_coarsening_annotation_remote_unit(annotation_remote_unit)
self.save_coarsening_remote_annotation_categories(remote_unit, annotation_remote_unit.remote_categories)
# set the task status to SUBMITTED
task.status = Constants.TASK_STATUS_JSON['SUBMITTED']
task.save()
print('save_task_by_layer_type - COARSENING - end')
def save_coarsening_remote_annotation_categories(self,remote_annotation_unit,categories):
print('save_remote_annotation_categories - start')
for cat in categories:
unit_category = Annotation_Units_Categories()
unit_category.unit_id = remote_annotation_unit.remote_unit_id
unit_category.category_id = cat.category_id
unit_category.remote_parent_id = remote_annotation_unit.unit_id
try:
unit_category.save()
except:
print('already saved remote category')
print('save_remote_annotation_categories - end')
def save_coarsening_annotation_remote_unit(self, annotation_unit):
print("save_coarsening_annotation_remote_unit - start")
remote_unit = Annotation_Remote_Units_Annotation_Units()
# remote_unit.unit_id means that it is the parent
remote_unit.unit_id = annotation_unit.parent_id
# remote_unit.remote_unit_id means that it is the remote unit
remote_unit_id = get_object_or_404(Annotation_Units,
tree_id=annotation_unit.tree_id,
task_id=annotation_unit.task_id)
remote_unit.remote_unit_id = remote_unit_id
remote_unit.save()
print("save_coarsening_annotation_remote_unit - end")
return remote_unit
def save_coarsening_annotation_categories(self,annotation_unit,parent_au,coarsenned_categories):
# add and duplicate all the parents annotation_units categories
if parent_au['categories'] is not None:
for cat in parent_au['categories']:
unit_category = Annotation_Units_Categories()
unit_category.unit_id = annotation_unit
unit_category.category_id = Categories.objects.get(id=cat['id'])
unit_category.remote_parent_id = None # TODO: can it be other then none ?
unit_category.save()
# if the category is the coarsened one, add it to the annotation unit
coarsend_category = self.get_coarsening_layer_category_or_none(coarsenned_categories, cat)
if coarsend_category is not None:
coarsend_unit_category = Annotation_Units_Categories()
coarsend_unit_category.unit_id = annotation_unit
coarsend_unit_category.category_id = coarsend_category
coarsend_unit_category.remote_parent_id = None # TODO: can it be other then none ?
try:
coarsend_unit_category.save()
except:
print("already saved")
def get_parent_annotation_unit_or_none(self,parent_au_id,task_id):
try:
return Annotation_Units.objects.get(tree_id=parent_au_id, task_id=task_id)
except:
return None
def get_coarsenned_categories(self,layer):
# changed_categories object = {"parent_category_id" : "into_category_id"}
changed_categories = {}
derived_layer_categories = Derived_Layers_Categories_Categories.objects.all().filter(layer_id=layer.id)
for d_l_cat in derived_layer_categories :
changed_categories[d_l_cat.parent_category_id_id] = d_l_cat.category_id_id
return changed_categories
def get_coarsening_layer_category_or_none(self,coarsenned_categories,cat):
try:
if coarsenned_categories[cat['id']] is not None:
cat_to_add = Categories.objects.get(id=coarsenned_categories[cat['id']])
# TODO: check if the cat_to_add already excsit in the parent_au
# parent_au['categories'].append(cat_to_add)
except:
# cat_to_add = Categories.objects.get(id=cat['id'])
cat_to_add = None
return cat_to_add
class TaskSerializerAnnotator(serializers.ModelSerializer):
created_by = DjangoUserSerializer_Simplify(many=False, read_only=True)
passage = PassageSerializer(many=False, read_only=True, allow_null=True)
project = ProjectSerializerForAnnotator(many=False, read_only=True, allow_null=False)
user = serializers.SerializerMethodField()
parent = serializers.SerializerMethodField()
children = serializers.SerializerMethodField()
tokens = serializers.SerializerMethodField()
annotation_units = serializers.SerializerMethodField()
is_active = serializers.SerializerMethodField()
user_comment = serializers.SerializerMethodField()
def get_is_active(self,obj):
if not obj.is_active:
raise GetForInactiveTaskException
return obj.is_active
def get_user(self,obj):
return DjangoUserSerializer_Simplify(obj.annotator).data
def get_parent(self,obj):
if obj.parent_task is not None:
return TaskInChartSerializer(obj.parent_task).data
else:
return None
def get_children(self, obj):
children_tasks = Tasks.objects.all().filter(parent_task_id=obj.id)
children_json = []
for cl in children_tasks:
children_json.append(TaskInChartSerializer(cl).data)
return children_json
def get_user_comment(self, obj):
if (obj.status == Constants.TASK_STATUS_JSON['NOT_STARTED'] and obj.type == Constants.TASK_TYPES_JSON['REVIEW']):
return obj.parent_task.user_comment
else:
return obj.user_comment
def get_tokens(self, obj):
data = None
if obj.annotation_json and obj.type != Constants.TASK_TYPES_JSON['TOKENIZATION']:
#logger.info("tokens_json detected")
#data = json.loads(obj.tokens_json.tokens_json)
data = json.loads(obj.annotation_json.annotation_json)
logger.info(str(data))
if len(data) > 0:
data = data[0]['children_tokens']
if not data:
logger.info("tokens_json not detected")
data = self._get_tokens(obj)
# tokenization tasks do not have a Tokens_Json
# There is no need for this, since the Tokens_Json is used for caching for quick GET in the annotation
# task.
#if obj.type != Constants.TASK_TYPES_JSON['TOKENIZATION']:
# tj = Tokens_Json.objects.create(task=obj, tokens_json=data_json)
# obj.tokens_json = tj
# obj.save()
return data
def _get_tokens(self, obj):
obj_tokens = Tokens.objects.all().filter(task_id=obj.id).order_by('start_index')
if (obj_tokens or obj.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
tokens = obj_tokens
elif obj.type == Constants.TASK_TYPES_JSON['REVIEW']:
tokens = []
parent_task = obj
while len(tokens) == 0:
tokens = Tokens.objects.all().filter(task_id=parent_task).order_by("start_index")
parent_task = parent_task.parent_task
else:
root_tokeniztion_task_id = self.get_root_task(obj)
tokens = Tokens.objects.all().filter(task_id=root_tokeniztion_task_id).order_by("start_index")
tokens_json = []
for index,t in enumerate(tokens):
cur_json = TokensSerializer(t,context={'index_in_task':index}).data
tokens_json.append(cur_json)
return tokens_json
def get_annotation_units(self, obj):
logger.info("get_annotation_units accessed")
# **********************************
# AS ARRAY
# **********************************
def create_annotation(obj):
# TODO: Place all this in an internal function
orig_obj = None
annotation_units = Annotation_Units.objects.all().filter(task_id=obj.id)
# handle new refinement or extention layer taks - get the parent annotation units - start
if (len(annotation_units) == 0 and obj.parent_task is not None): # TODO: check if coarsening task is ok with that
# get the parent task annotation units
orig_obj = obj
obj = obj.parent_task
annotation_units = Annotation_Units.objects.all().filter(task_id=obj.id)
annotation_units = annotation_units.select_related('parent_id')
# handle new refinement or extention layer taks - get the parent annotation units - end
annotation_units_json = []
remote_annotation_unit_array = []
for au in annotation_units:
# set as default is_remote_copy = False
au.is_remote_copy = False
# check if i have a remote units
remote_units = Annotation_Remote_Units_Annotation_Units.objects.filter(unit_id=au).select_related(
'remote_unit_id')
for ru in remote_units:
# retrieve its original unit
remote_original_unit = ru.remote_unit_id # Annotation_Units.objects.get(id = ru.remote_unit_id.id, task_id=obj.id)
# set the remote is_remote_copy = true
remote_original_unit.is_remote_copy = True
# set the parent_id to be the remote's one
remote_original_unit.parent_id = ru.unit_id
# setting the cloned_from tree_id
cloned_from_tree_id = remote_original_unit.tree_id
# set the tree_id to be that of the remote unit
remote_original_unit.tree_id = ru.remote_unit_tree_id
# add the remote original unit to the json output
annotation_units_json.append(Annotation_UnitsSerializer(remote_original_unit, context={
'cloned_from_tree_id': cloned_from_tree_id}).data)
au_data = Annotation_UnitsSerializer(au).data
if (orig_obj and orig_obj.project.layer.type != Constants.LAYER_TYPES_JSON['ROOT']):
# take Annotation_UnitsSerializer(au).data, and alter slot to be 3+
for index, cat in enumerate(au_data['categories']):
au_data['categories'][index]['slot'] = 3 + index
annotation_units_json.append(au_data)
# return all array sorted with all the remote units in the end
annotation_units_json.sort(key=lambda x: tuple([int(a) for a in x['tree_id'].split('-')]))
return annotation_units_json
if obj.annotation_json:
data = json.loads(obj.annotation_json.annotation_json)
else:
data = create_annotation(obj)
data_json = json.dumps(data)
aj = Annotation_Json.objects.create(task=obj, annotation_json=data_json)
obj.annotation_json = aj
obj.save()
return data
#return sorted(annotation_units_json, key=operator.itemgetter('is_remote_copy'), reverse=False)
# **********************************
# AS ROOT OBJECT
# **********************************
# try:
# au = Annotation_Units.objects.get(task_id_id=obj.id, parent_id=None)
# except Annotation_Units.DoesNotExist:
# au = None
# return Annotation_UnitsSerializer(au).data
def get_root_task(self,task_instance):
root_task = task_instance
while (root_task.parent_task != None ):
root_task = root_task.parent_task
return root_task.id
class Meta:
model = Tasks
fields = (
'id',
'parent',
'children',
'type',
'project',
'user',
'passage',
'tokens',
'annotation_units',
'is_demo',
'manager_comment',
'user_comment',
'is_active',
'created_by',
'created_at',
'updated_at'
)
def update(self, instance, validated_data):
if instance.status == 'SUBMITTED':
raise CantChangeSubmittedTaskExeption
save_type = self.context['save_type']
if(save_type == 'draft'):
self.save_draft(instance)
elif (save_type == 'submit'):
self.submit(instance)
elif (save_type == 'reset'):
self.reset(instance)
return instance
def reset(self,instance):
# TODO: Clear instance.annotation_json
instance.annotation_json = None
instance.tokens_json = None
instance.status = Constants.TASK_STATUS_JSON['NOT_STARTED']
instance.user_comment = ''
if (instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
self.reset_tokenization_task(instance)
else:
self.reset_current_task(instance)
instance.save()
def save_draft(self,instance):
instance.status = 'ONGOING'
print('save_draft')
if (instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
self.save_tokenization_task(instance)
elif instance.type in [Constants.TASK_TYPES_JSON['ANNOTATION'],Constants.TASK_TYPES_JSON['REVIEW']]:
self.validate_annotation_task(instance)
data_json = json.dumps(self.initial_data['annotation_units'])
aj = Annotation_Json.objects.create(task=instance, annotation_json=data_json)
instance.annotation_json = aj
tokens_json = json.dumps(self.initial_data['tokens'])
tj = Tokens_Json.objects.create(task=instance, tokens_json=tokens_json)
instance.tokens_json = tj
instance.user_comment = self.initial_data['user_comment']
# elif (instance.type == Constants.TASK_TYPES_JSON['REVIEW']):
# self.validate_annotation_task(instance)
# data_json = json.dumps(self.initial_data['annotation_units'])
# aj = Annotation_Json.objects.create(task=instance, annotation_json=data_json)
# instance.annotation_json = aj
# tokens_json = json.dumps(self.initial_data['tokens'])
# tj = Tokens_Json.objects.create(task=instance, tokens_json=tokens_json)
# instance.tokens_json = tj
# instance.user_comment = self.initial_data['user_comment']
instance.save()
def reset_tokenization_task(self,instance):
self.check_if_parent_task_ok_or_exception(instance)
instance.tokens_set.all().delete()
def reset_annotation_task(self,instance):
self.check_if_parent_task_ok_or_exception(instance)
instance.tokens_set.all().delete()
def save_tokens(self, instance):
instance.tokens_set.all().delete()
tokens = []
old_to_new_token_id_map = dict()
for token in self.initial_data['tokens']:
newToken = Tokens()
newToken.task_id_id = instance
newToken.text = token['text']
newToken.require_annotation = not isPunct(newToken.text)
newToken.start_index = token['start_index']
newToken.end_index = token['end_index']
instance.tokens_set.add(newToken,bulk=False)
old_to_new_token_id_map[token['id']] = newToken.id
tokens.append(newToken)
return tokens, old_to_new_token_id_map
def save_tokenization_task(self,instance):
print('save_tokenization_task - start')
self.check_if_parent_task_ok_or_exception(instance)
instance.tokens_set.all().delete()
for token in self.initial_data['tokens']:
newToken = Tokens()
newToken.task_id_id = instance
newToken.text = token['text']
newToken.require_annotation = not isPunct(newToken.text)
newToken.start_index = token['start_index']
newToken.end_index = token['end_index']
instance.tokens_set.add(newToken,bulk=False)
print('save_tokenization_task - end')
# def save_annotation_task(self, instance):
# # TODO: Split into validate_annotation_task and save_annotation_task
# # validate_annotation_task only validates the initial_data without reading or writing to the database
# # self.initial_data is the JSON received from the frontend
# print('save_annotation_task - start')
# logger.info('save_annotation_task - start')
#
# # mainly saving an annotations units array
# self.check_if_parent_task_ok_or_exception(instance) # Validation
# self.reset_current_task(instance) # DB
# remote_units_array = []
# instance.user_comment = self.initial_data['user_comment'] # DB
#
# # validating tokens
# tokens = self.initial_data['tokens']
# if not strictly_increasing([x['start_index'] for x in tokens]):
# raise TokensInvalid("tokens should be ordered by their start_index")
# tokens_id_to_startindex = dict([(x['id'], x['start_index']) for x in tokens])
# children_tokens_list_for_validation = []
# for au in self.initial_data['annotation_units']:
# cur_children_tokens = au.get('children_tokens')
# try:
# if cur_children_tokens:
# cur_children_tokens_start_indices = [tokens_id_to_startindex[x['id']] for x in cur_children_tokens]
# else:
# if au['type'] == 'IMPLICIT' or au['tree_id'] == '0':
# cur_children_tokens_start_indices = None
# else:
# raise TokensInvalid("Only implicit units may not have a children_tokens field")
# except KeyError:
# raise TokensInvalid("children_tokens contains a token which is not in the task's tokens list.")
# children_tokens_list_for_validation.append((au['tree_id'],(au['parent_tree_id'],au['is_remote_copy'],cur_children_tokens_start_indices)))
#
#
# print("children_tokens_list_for_validation: "+str(cur_children_tokens_start_indices))
# if not check_children_tokens(children_tokens_list_for_validation):
# raise TokensInvalid("Inconsistency in children_tokens detected.")
#
# all_tree_ids = [] # a list of all tree_ids by their order in the input
#
# annotation_unit_map = {} # tree_id -> annotation_unit object
#
# for au in self.initial_data['annotation_units']:
# annotation_unit = Annotation_Units()
# if is_correct_format_tree_id(au['tree_id']):
# annotation_unit.tree_id = au['tree_id']
# all_tree_ids.append(au['tree_id'])
# annotation_unit_map[annotation_unit.tree_id] = annotation_unit
# else:
# raise TreeIdInvalid("tree_id is in an incorrect format; fix unit " + str(annotation_unit.tree_id))
#
# annotation_unit.task_id = instance
# if au['type'] in [x[0] for x in Constants.ANNOTATION_UNIT_TYPES]:
# annotation_unit.type = au['type']
# else:
# raise UnallowedValueError("An annotation unit is given an unallowed type: "+au['type'])
#
# annotation_unit.comment = au['comment']
# annotation_unit.cluster = au['cluster']
#
# annotation_unit.is_remote_copy = au['is_remote_copy']
#
# parent_id = None
# if au['parent_tree_id']:
# if not is_correct_format_tree_id(au['parent_tree_id']):
# raise TreeIdInvalid("parent_tree_id is in an incorrect format; fix unit "+str(annotation_unit.tree_id))
# if not is_correct_format_tree_id_child(au['parent_tree_id'],au['tree_id']):
# raise TreeIdInvalid("parent_tree_id and tree_id do not match in format; fix unit " + str(annotation_unit.tree_id))
#
# # parent_id = get_object_or_404(Annotation_Units, tree_id=au['parent_tree_id'],task_id=instance.id)
# parent_id = annotation_unit_map[au['parent_tree_id']]
# else:
# if annotation_unit.tree_id != '0':
# raise TreeIdInvalid("All annotation units but unit 0 must have a valid, non-null tree_id; fix unit "+str(annotation_unit.tree_id))
#
# annotation_unit.parent_id = parent_id
# annotation_unit.gui_status = au['gui_status']
#
# if annotation_unit.is_remote_copy:
#
# annotation_unit.remote_categories = get_value_or_none('categories', au)
# if au['cloned_from_tree_id']:
# if not is_correct_format_tree_id(au['cloned_from_tree_id']):
# raise TreeIdInvalid("cloned_from_tree_id is in an incorrect format; fix unit " + str(
# annotation_unit.tree_id))
# annotation_unit.cloned_from_tree_id = au['cloned_from_tree_id']
# else:
# raise TreeIdInvalid("cloned_from_tree_id should be defined for all remote units")
# remote_units_array.append(annotation_unit)
# else: # not a remote unit
# if au['cloned_from_tree_id']:
# raise TreeIdInvalid("cloned_from_tree_id should not be defined for non-remote units")
# instance.annotation_units_set.add(annotation_unit,bulk=False)
# # The following two functions just save data and do not validate anything
# self.save_children_tokens(annotation_unit, get_value_or_none('children_tokens', au),tokens_id_to_startindex)
# self.save_annotation_categories(annotation_unit, get_value_or_none('categories', au))
#
# if not is_tree_ids_uniq_and_consecutive(all_tree_ids):
# raise TreeIdInvalid("tree_ids within a unit should be unique and consecutive")
#
# for annotation_unit in remote_units_array:
# # TODO: Check if these functions do any validation
# remote_unit = self.save_annotation_remote_unit(annotation_unit)
# self.save_remote_annotation_categories(remote_unit,annotation_unit.remote_categories)
#
# print('save_annotation_task - end')
# logger.info('save_annotation_task - end')
def save_annotation_task(self, instance):
# self.initial_data is the JSON received from the frontend
print('save_annotation_task - start')
logger.info('save_annotation_task - start')
# mainly saving an annotations units array
self.reset_current_task(instance)
remote_units_array = []
instance.user_comment = self.initial_data['user_comment']
# save the new tokens instead of the old ones
tokens, old_to_new_token_id_map= self.save_tokens(instance)
all_tree_ids = [] # a list of all tree_ids by their order in the input
annotation_unit_map = {} # tree_id -> annotation_unit object
for au in self.initial_data['annotation_units']:
annotation_unit = Annotation_Units()
if is_correct_format_tree_id(au['tree_id']):
annotation_unit.tree_id = au['tree_id']
all_tree_ids.append(au['tree_id'])
annotation_unit_map[annotation_unit.tree_id] = annotation_unit
annotation_unit.task_id = instance
if au['type'] in [x[0] for x in Constants.ANNOTATION_UNIT_TYPES]:
annotation_unit.type = au['type']
annotation_unit.comment = au['comment']
annotation_unit.cluster = au['cluster']
annotation_unit.is_remote_copy = au['is_remote_copy']
# parent_id = get_object_or_404(Annotation_Units, tree_id=au['parent_tree_id'],task_id=instance.id)
parent_id = annotation_unit_map[au['parent_tree_id']] if au['parent_tree_id'] else None
annotation_unit.parent_id = parent_id
annotation_unit.gui_status = au['gui_status']
if annotation_unit.is_remote_copy:
annotation_unit.remote_categories = get_value_or_none('categories', au)
if au['cloned_from_tree_id']:
annotation_unit.cloned_from_tree_id = au['cloned_from_tree_id']
remote_units_array.append(annotation_unit)
else: # not a remote unit
instance.annotation_units_set.add(annotation_unit, bulk=False)
self.save_children_tokens(annotation_unit, get_value_or_none('children_tokens', au), old_to_new_token_id_map)
self.save_annotation_categories(annotation_unit, get_value_or_none('categories', au))
for annotation_unit in remote_units_array:
remote_unit = self.save_annotation_remote_unit(annotation_unit)
self.save_remote_annotation_categories(remote_unit, annotation_unit.remote_categories)
instance.save()
print('save_annotation_task - end')
logger.info('save_annotation_task - end')
def validate_annotation_task(self, instance):
# validate_annotation_task only validates the initial_data without reading or writing to the database
# self.initial_data is the JSON received from the frontend
print('validate_annotation_task - start')
logger.info('validate_annotation_task - start')
if not self.initial_data['id'] == instance.id:
raise DiscrepancyBetweenTaskIdsException('Task id must me the same, of the instance and of the initial data')
self.check_if_parent_task_ok_or_exception(instance)
# validating tokens
tokens = self.initial_data['tokens']
if not is_increasing([x['start_index'] for x in tokens],strict=True):
raise TokensInvalid("tokens should be ordered by their start_index")
tokens_id_to_startindex = dict([(x['id'], x['start_index']) for x in tokens])
children_tokens_list_for_validation = []
largest_index_in_task_tokens = self.initial_data['tokens'][-1]['start_index']
for au in self.initial_data['annotation_units']:
cur_children_tokens = au.get('children_tokens')
try:
if cur_children_tokens:
start_indices = \
[tokens_id_to_startindex[children_token['id']] for children_token in cur_children_tokens]
if any(start_index > largest_index_in_task_tokens for start_index in start_indices):
raise TokensInvalid("Invalid start index in unit %s, larger then the biggest token" % au['tree_id'])
if len(start_indices) > len(set(start_indices)):
raise TokensInvalid("Duplicate start index in children tokens in unit %s" % au['tree_id'])
cur_children_tokens_start_indices = [tokens_id_to_startindex[x['id']] for x in cur_children_tokens]
else:
if au['type'] == 'IMPLICIT':
cur_children_tokens_start_indices = None
else:
raise TokensInvalid("Only implicit units may not have a children_tokens field. Annotation unit %s does not contain children_tokens" %au['tree_id'])
except KeyError:
raise TokensInvalid("children_tokens contains a token which is not in the task's tokens list.")
children_tokens_list_for_validation.append(
(au['tree_id'], (au['parent_tree_id'], au['is_remote_copy'], cur_children_tokens_start_indices)))
if au['parent_tree_id']:
if not is_correct_format_tree_id(au['parent_tree_id']):
raise TreeIdInvalid(
"parent_tree_id is in an incorrect format; fix unit " + str(au['tree_id']))
if not is_correct_format_tree_id_child(au['parent_tree_id'], au['tree_id']):
raise TreeIdInvalid(
"parent_tree_id and tree_id do not match in format; fix unit " + str(au['tree_id']))
else:
if au['tree_id'] != '0':
raise TreeIdInvalid(
"All annotation units but unit 0 must have a valid, non-null tree_id; fix unit " + str(
au['tree_id']))
print("children_tokens_list_for_validation: " + str(cur_children_tokens_start_indices))
if not check_children_tokens(children_tokens_list_for_validation): # will never happens
raise TokensInvalid("Inconsistency in children_tokens detected.")
all_tree_ids = [] # a list of all tree_ids by their order in the input
for au in self.initial_data['annotation_units']:
if is_correct_format_tree_id(au['tree_id']):
all_tree_ids.append(au['tree_id'])
else:
raise TreeIdInvalid("tree_id is in an incorrect format; fix unit " + str(au['tree_id']))
if not (au['type'] in [x[0] for x in Constants.ANNOTATION_UNIT_TYPES]):
raise UnallowedValueError("An annotation unit is given an unallowed type: " + au['type'])
# if au['type'] == 'IMPLICIT' and au['gui_status'] != "OPEN":
# raise UnallowedValueError("Remotes and implicit units must have a gui_status OPEN")
if au['is_remote_copy']:
if au['cloned_from_tree_id']:
if not is_correct_format_tree_id(au['cloned_from_tree_id']):
raise TreeIdInvalid("cloned_from_tree_id is in an incorrect format; fix unit " + str(
au['tree_id']))
else:
raise TreeIdInvalid("cloned_from_tree_id should be defined for all remote units")
else: # not a remote unit
if au['cloned_from_tree_id']:
raise TreeIdInvalid("cloned_from_tree_id should not be defined for non-remote units")
# OMRI: buggy
#if au['gui_status'] == 'HIDDEN' and '-' in au['tree_id']:
# raise TreeIdInvalid("annotation unit " + str(au['tree_id']) + " has HIDDEN gui status, should not be an internal unit")
#if au['is_remote_copy'] or au['type'] == 'IMPLICIT':
# if au['gui_status'] != 'OPEN':
# raise RemoteIsNotOpen('remote or implicit unit ' + str(au['tree_id']) + ' should have an OPEN gui status')
if not is_tree_ids_uniq_and_consecutive(all_tree_ids):
raise TreeIdInvalid("tree_ids within a unit should be unique and consecutive")
print('validate_annotation_task - end')
logger.info('validate_annotation_task - end')
def save_remote_annotation_categories(self,remote_annotation_unit,categories):
print('save_remote_annotation_categories - start')
for cat in categories:
unit_category = Annotation_Units_Categories()
unit_category.unit_id = remote_annotation_unit.remote_unit_id
unit_category.category_id = Categories.objects.get(id=cat['id'])
unit_category.remote_parent_id = remote_annotation_unit.unit_id
# Omri added Sep 12:
if 'slot' in cat: # Omri TODO: disallow the option not to specify a slot
unit_category.slot = cat['slot']
else:
unit_category.slot = 1
unit_category.save()
print('save_remote_annotation_categories - end')
def reset_current_task(self,task_instance):
# TODO: validate the new array of annotation units before deleting the current one
print('reset_current_task - start')
# reset Annotation_Units_Tokens
# reset Annotation_Units_Categories
# reset Annotation_Remote_Units_Annotation_Units
# reset annotaion_units
task_instance.annotation_units_set.all().delete()
print('reset_current_task - end')
def save_annotation_remote_unit(self,annotation_unit):
remote_unit = Annotation_Remote_Units_Annotation_Units()
# remote_unit.unit_id means that it is the parent
remote_unit.unit_id = annotation_unit.parent_id
# remote_unit.remote_unit_id means that it is the unit it was cloned from
remote_unit_id = get_object_or_404(Annotation_Units, tree_id=annotation_unit.cloned_from_tree_id, task_id=annotation_unit.task_id )
remote_unit.remote_unit_id = remote_unit_id
# saving the tree_id of the remote unit
remote_unit.remote_unit_tree_id =annotation_unit.tree_id
remote_unit.save()
return remote_unit
def save_children_tokens(self,annotation_unit,tokens,old_to_new_token_id_map):
if tokens != None:
annotation_units = [Annotation_Units_Tokens(unit_id=annotation_unit, token_id_id = old_to_new_token_id_map[t['id']]) for t in tokens]
Annotation_Units_Tokens.objects.bulk_create(annotation_units)
# for t in tokens:
# annotation_units_token = Annotation_Units_Tokens()
# annotation_units_token.unit_id = annotation_unit
# # annotation_units_token.token_id = Tokens.objects.get(id=t['id'])
# annotation_units_token.token_id_id = t['id']
# annotation_units_token.save()
def save_annotation_categories(self,annotation_unit,categories):
print('save_annotation_categories - start')
unit_categories = []
for cat in categories:
unit_category = Annotation_Units_Categories()
unit_category.unit_id = annotation_unit
# unit_category.category_id = Categories.objects.get(id=cat['id'])
unit_category.category_id_id = cat['id']
if 'slot' in cat: # Omri TODO: disallow the option not to specify a slot
unit_category.slot = cat['slot']
else:
unit_category.slot = 1
unit_category.remote_parent_id = None
unit_categories.append(unit_category)
# unit_category.save()
Annotation_Units_Categories.objects.bulk_create(unit_categories)
print('save_annotation_categories - end')
def save_review_task(self,instance):
# TODO: CHECK IF OK !!!!
print('save_review_task - start')
self.save_annotation_task(instance)
print('save_review_task - end')
def submit(self,instance):
if instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']:
self.save_tokenization_task(instance)
elif (instance.type == Constants.TASK_TYPES_JSON['ANNOTATION']):
self.validate_annotation_task(instance)
self.save_annotation_task(instance)
elif (instance.type == Constants.TASK_TYPES_JSON['REVIEW']):
self.validate_annotation_task(instance)
self.save_review_task(instance)
instance.status = 'SUBMITTED'
instance.save(update_fields=['status'])
def check_if_parent_task_ok_or_exception(self,instance):
if instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']:
if instance.parent_task != None:
raise SaveTaskTypeDeniedException
elif instance.parent_task == None:
raise SaveTaskTypeDeniedException
class TaskSerializerAnnotator(serializers.ModelSerializer):
created_by = DjangoUserSerializer_Simplify(many=False, read_only=True)
passage = PassageSerializer(many=False, read_only=True, allow_null=True)
project = ProjectSerializerForAnnotator(many=False,
read_only=True,
allow_null=False)
user = serializers.SerializerMethodField()
parent = serializers.SerializerMethodField()
children = serializers.SerializerMethodField()
tokens = serializers.SerializerMethodField()
annotation_units = serializers.SerializerMethodField()
is_active = serializers.SerializerMethodField()
def get_is_active(self, obj):
if not obj.is_active:
raise GetForInactiveTaskException
return obj.is_active
def get_user(self, obj):
return DjangoUserSerializer_Simplify(obj.annotator).data
def get_parent(self, obj):
if obj.parent_task is not None:
return TaskInChartSerializer(obj.parent_task).data
else:
return None
def get_children(self, obj):
children_tasks = Tasks.objects.all().filter(parent_task_id=obj.id)
children_json = []
for cl in children_tasks:
children_json.append(TaskInChartSerializer(cl).data)
return children_json
def get_tokens(self, obj):
if (obj.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
tokens = Tokens.objects.all().filter(
task_id=obj.id).order_by('start_index')
else:
# get the tokens array from the root tokenization task
root_tokeniztion_task_id = self.get_root_task(obj)
tokens = Tokens.objects.all().filter(
task_id=root_tokeniztion_task_id).order_by("start_index")
tokens_json = []
for index, t in enumerate(tokens):
cur_json = TokensSerializer(t, context={
'index_in_task': index
}).data
#cur_json['index_in_task'] = index
tokens_json.append(cur_json)
return tokens_json
def get_annotation_units(self, obj):
logger.info("get_annotation_units accessed")
# **********************************
# AS ARRAY
# **********************************
orig_obj = None
annotation_units = Annotation_Units.objects.all().filter(
task_id=obj.id)
# handle new refinement or extention layer taks - get the parent annotation units - start
if (len(annotation_units) == 0 and obj.parent_task
is not None): # TODO: check if coarsening task is ok with that
# get the parent task annotation units
orig_obj = obj
obj = obj.parent_task
annotation_units = Annotation_Units.objects.all().filter(
task_id=obj.id)
# handle new refinement or extention layer taks - get the parent annotation units - end
annotation_units_json = []
remote_annotation_unit_array = []
for au in annotation_units:
# set as default is_remote_copy = False
au.is_remote_copy = False
# check if i have a remote units
remote_units = Annotation_Remote_Units_Annotation_Units.objects.all(
).filter(unit_id=au)
for ru in remote_units:
# retrieve its original unit
remote_original_unit = Annotation_Units.objects.get(
id=ru.remote_unit_id.id, task_id=obj.id)
# set the remote is_remote_copy = true
remote_original_unit.is_remote_copy = True
# set the parent_id to be the remote's one
remote_original_unit.parent_id = ru.unit_id
# setting the cloned_from tree_id
cloned_from_tree_id = remote_original_unit.tree_id
# set the tree_id to be that of the remote unit
remote_original_unit.tree_id = ru.remote_unit_tree_id
# add the remote original unit to the json output
annotation_units_json.append(
Annotation_UnitsSerializer(remote_original_unit,
context={
'cloned_from_tree_id':
cloned_from_tree_id
}).data)
au_data = Annotation_UnitsSerializer(au).data
if (orig_obj and orig_obj.project.layer.type !=
Constants.LAYER_TYPES_JSON['ROOT']):
# take Annotation_UnitsSerializer(au).data, and alter slot to be 3+
for index, cat in enumerate(au_data['categories']):
au_data['categories'][index]['slot'] = 3 + index
annotation_units_json.append(au_data)
# return all array sorted with all the remote units in the end
annotation_units_json.sort(
key=lambda x: tuple([int(a) for a in x['tree_id'].split('-')]))
return annotation_units_json
#return sorted(annotation_units_json, key=operator.itemgetter('is_remote_copy'), reverse=False)
# **********************************
# AS ROOT OBJECT
# **********************************
# try:
# au = Annotation_Units.objects.get(task_id_id=obj.id, parent_id=None)
# except Annotation_Units.DoesNotExist:
# au = None
# return Annotation_UnitsSerializer(au).data
def get_root_task(self, task_instance):
root_task = task_instance
while (root_task.parent_task != None):
root_task = root_task.parent_task
return root_task.id
class Meta:
model = Tasks
fields = ('id', 'parent', 'children', 'type', 'project', 'user',
'passage', 'tokens', 'annotation_units', 'is_demo',
'manager_comment', 'user_comment', 'is_active', 'created_by',
'created_at', 'updated_at')
def update(self, instance, validated_data):
if instance.status == 'SUBMITTED':
raise CantChangeSubmittedTaskExeption
save_type = self.context['save_type']
if (save_type == 'draft'):
self.save_draft(instance)
elif (save_type == 'submit'):
self.submit(instance)
elif (save_type == 'reset'):
self.reset(instance)
return instance
def reset(self, instance):
instance.status = Constants.TASK_STATUS_JSON['NOT_STARTED']
instance.user_comment = ''
if (instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
self.reset_tokenization_task(instance)
else:
self.reset_current_task(instance)
instance.save()
def save_draft(self, instance):
instance.status = 'ONGOING'
print('save_draft')
if (instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
self.save_tokenization_task(instance)
elif (instance.type == Constants.TASK_TYPES_JSON['ANNOTATION']):
self.save_annotation_task(instance)
elif (instance.type == Constants.TASK_TYPES_JSON['REVIEW']):
self.save_review_task(instance)
instance.save()
def reset_tokenization_task(self, instance):
self.check_if_parent_task_ok_or_exception(instance)
instance.tokens_set.all().delete()
def reset_annotation_task(self, instance):
self.check_if_parent_task_ok_or_exception(instance)
instance.tokens_set.all().delete()
def save_tokenization_task(self, instance):
print('save_tokenization_task - start')
self.check_if_parent_task_ok_or_exception(instance)
instance.tokens_set.all().delete()
for token in self.initial_data['tokens']:
newToken = Tokens()
newToken.task_id_id = instance
newToken.text = token['text']
newToken.require_annotation = token['require_annotation']
newToken.start_index = token['start_index']
newToken.end_index = token['end_index']
instance.tokens_set.add(newToken, bulk=False)
print('save_tokenization_task - end')
def save_annotation_task(self, instance):
print('save_annotation_task - start')
# mainly saving an annotations units array
self.check_if_parent_task_ok_or_exception(instance)
self.reset_current_task(instance)
remote_units_array = []
instance.user_comment = self.initial_data['user_comment']
# validating tokens
tokens = self.initial_data['tokens']
if not strictly_increasing([x['start_index'] for x in tokens]):
raise TokensInvalid(
"tokens should be ordered by their start_index")
tokens_id_to_startindex = dict([(x['id'], x['start_index'])
for x in tokens])
children_tokens_list_for_validation = []
for au in self.initial_data['annotation_units']:
cur_children_tokens = au.get('children_tokens')
try:
if cur_children_tokens:
cur_children_tokens_start_indices = [
tokens_id_to_startindex[x['id']]
for x in cur_children_tokens
]
else:
if au['type'] == 'IMPLICIT' or au['tree_id'] == '0':
cur_children_tokens_start_indices = None
else:
raise TokensInvalid(
"Only implicit units may not have a children_tokens field"
)
except KeyError:
raise TokensInvalid(
"children_tokens contains a token which is not in the task's tokens list."
)
children_tokens_list_for_validation.append(
(au['tree_id'], (au['parent_tree_id'], au['is_remote_copy'],
cur_children_tokens_start_indices)))
print("children_tokens_list_for_validation: " +
str(cur_children_tokens_start_indices))
if not check_children_tokens(children_tokens_list_for_validation):
raise TokensInvalid("Inconsistency in children_tokens detected.")
all_tree_ids = [] # a list of all tree_ids by their order in the input
for au in self.initial_data['annotation_units']:
annotation_unit = Annotation_Units()
if is_correct_format_tree_id(au['tree_id']):
annotation_unit.tree_id = au['tree_id']
all_tree_ids.append(au['tree_id'])
else:
raise TreeIdInvalid(
"tree_id is in an incorrect format; fix unit " +
str(annotation_unit.tree_id))
annotation_unit.task_id = instance
if au['type'] in [x[0] for x in Constants.ANNOTATION_UNIT_TYPES]:
annotation_unit.type = au['type']
else:
raise UnallowedValueError(
"An annotation unit is given an unallowed type: " +
au['type'])
annotation_unit.comment = au['comment']
annotation_unit.cluster = au['cluster']
annotation_unit.is_remote_copy = au['is_remote_copy']
parent_id = None
if au['parent_tree_id']:
if not is_correct_format_tree_id(au['parent_tree_id']):
raise TreeIdInvalid(
"parent_tree_id is in an incorrect format; fix unit " +
str(annotation_unit.tree_id))
if not is_correct_format_tree_id_child(au['parent_tree_id'],
au['tree_id']):
raise TreeIdInvalid(
"parent_tree_id and tree_id do not match in format; fix unit "
+ str(annotation_unit.tree_id))
parent_id = get_object_or_404(Annotation_Units,
tree_id=au['parent_tree_id'],
task_id=instance.id)
else:
if annotation_unit.tree_id != '0':
raise TreeIdInvalid(
"All annotation units but unit 0 must have a valid, non-null tree_id; fix unit "
+ str(annotation_unit.tree_id))
annotation_unit.parent_id = parent_id
annotation_unit.gui_status = au['gui_status']
if annotation_unit.is_remote_copy:
annotation_unit.remote_categories = get_value_or_none(
'categories', au)
if au['cloned_from_tree_id']:
if not is_correct_format_tree_id(
au['cloned_from_tree_id']):
raise TreeIdInvalid(
"cloned_from_tree_id is in an incorrect format; fix unit "
+ str(annotation_unit.tree_id))
annotation_unit.cloned_from_tree_id = au[
'cloned_from_tree_id']
else:
raise TreeIdInvalid(
"cloned_from_tree_id should be defined for all remote units"
)
remote_units_array.append(annotation_unit)
else: # not a remote unit
if au['cloned_from_tree_id']:
raise TreeIdInvalid(
"cloned_from_tree_id should not be defined for non-remote units"
)
instance.annotation_units_set.add(annotation_unit, bulk=False)
self.save_children_tokens(
annotation_unit, get_value_or_none('children_tokens', au),
tokens_id_to_startindex)
self.save_annotation_categories(
annotation_unit, get_value_or_none('categories', au))
if not is_tree_ids_uniq_and_consecutive(all_tree_ids):
raise TreeIdInvalid(
"tree_ids within a unit should be unique and consecutive")
for annotation_unit in remote_units_array:
remote_unit = self.save_annotation_remote_unit(annotation_unit)
self.save_remote_annotation_categories(
remote_unit, annotation_unit.remote_categories)
print('save_annotation_task - end')
def save_remote_annotation_categories(self, remote_annotation_unit,
categories):
print('save_remote_annotation_categories - start')
for cat in categories:
unit_category = Annotation_Units_Categories()
unit_category.unit_id = remote_annotation_unit.remote_unit_id
unit_category.category_id = Categories.objects.get(id=cat['id'])
unit_category.remote_parent_id = remote_annotation_unit.unit_id
# Omri added Sep 12:
if 'slot' in cat: # Omri TODO: disallow the option not to specify a slot
unit_category.slot = cat['slot']
else:
unit_category.slot = 1
unit_category.save()
print('save_remote_annotation_categories - end')
def reset_current_task(self, task_instance):
# TODO: validate the new array of annotation units before deleting the current one
print('reset_current_task - start')
# reset Annotation_Units_Tokens
# reset Annotation_Units_Categories
# reset Annotation_Remote_Units_Annotation_Units
# reset annotaion_units
task_instance.annotation_units_set.all().delete()
print('reset_current_task - end')
def save_annotation_remote_unit(self, annotation_unit):
remote_unit = Annotation_Remote_Units_Annotation_Units()
# remote_unit.unit_id means that it is the parent
remote_unit.unit_id = annotation_unit.parent_id
# remote_unit.remote_unit_id means that it is the unit it was cloned from
remote_unit_id = get_object_or_404(
Annotation_Units,
tree_id=annotation_unit.cloned_from_tree_id,
task_id=annotation_unit.task_id)
remote_unit.remote_unit_id = remote_unit_id
# saving the tree_id of the remote unit
remote_unit.remote_unit_tree_id = annotation_unit.tree_id
remote_unit.save()
return remote_unit
def save_children_tokens(self, annotation_unit, tokens, id_to_start_index):
if tokens != None:
print('save_children_tokens - start')
for t in tokens:
annotation_units_token = Annotation_Units_Tokens()
annotation_units_token.unit_id = annotation_unit
annotation_units_token.token_id = Tokens.objects.get(
id=t['id'])
annotation_units_token.save()
print('save_children_tokens - end')
def save_annotation_categories(self, annotation_unit, categories):
print('save_annotation_categories - start')
for cat in categories:
unit_category = Annotation_Units_Categories()
unit_category.unit_id = annotation_unit
unit_category.category_id = Categories.objects.get(id=cat['id'])
if 'slot' in cat: # Omri TODO: disallow the option not to specify a slot
unit_category.slot = cat['slot']
else:
unit_category.slot = 1
unit_category.remote_parent_id = None
unit_category.save()
print('save_annotation_categories - end')
def save_review_task(self, instance):
# TODO: CHECK IF OK !!!!
print('save_review_task - start')
self.save_annotation_task(instance)
print('save_review_task - end')
def submit(self, instance):
if instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']:
self.save_tokenization_task(instance)
instance.status = 'SUBMITTED'
instance.save(update_fields=['status'])
def check_if_parent_task_ok_or_exception(self, instance):
if instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']:
if instance.parent_task != None:
raise SaveTaskTypeDeniedException
elif instance.parent_task == None:
raise SaveTaskTypeDeniedException
class TaskSerializerAnnotator(serializers.ModelSerializer):
created_by = DjangoUserSerializer_Simplify(many=False, read_only=True)
passage = PassageSerializer(many=False, read_only=True, allow_null=True)
project = ProjectSerializerForAnnotator(many=False,
read_only=True,
allow_null=False)
user = serializers.SerializerMethodField()
parent = serializers.SerializerMethodField()
children = serializers.SerializerMethodField()
tokens = serializers.SerializerMethodField()
annotation_units = serializers.SerializerMethodField()
def get_user(self, obj):
return DjangoUserSerializer_Simplify(obj.annotator).data
def get_parent(self, obj):
if obj.parent_task is not None:
return TaskInChartSerializer(obj.parent_task).data
else:
return None
def get_children(self, obj):
children_tasks = Tasks.objects.all().filter(parent_task_id=obj.id)
children_json = []
for cl in children_tasks:
children_json.append(TaskInChartSerializer(cl).data)
return children_json
def get_tokens(self, obj):
if (obj.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
tokens = Tokens.objects.all().filter(task_id=obj.id)
else:
# get the tokens array from the root tokenization task
root_tokeniztion_task_id = self.get_root_task(obj)
tokens = Tokens.objects.all().filter(
task_id=root_tokeniztion_task_id)
tokens_json = []
for t in tokens:
tokens_json.append(TokensSerializer(t).data)
tokens_json.sort(key=lambda x: x["start_index"])
return tokens_json
def get_annotation_units(self, obj):
# **********************************
# AS ARRAY
# **********************************
annotation_units = Annotation_Units.objects.all().filter(
task_id=obj.id).order_by('id')
# handle new refinement or extention layer taks - get the parent annotation units - start
if (len(annotation_units) == 0 and obj.parent_task
is not None): # TODO: check if coarsening task is ok with that
# get the parent task annotation units
obj = obj.parent_task
annotation_units = Annotation_Units.objects.all().filter(
task_id=obj.id).order_by('id')
# handle new refinement or extention layer taks - get the parent annotation units - end
annotation_units_json = []
remote_annotation_unit_array = []
for au in annotation_units:
# set as default is_remote_copy = False
au.is_remote_copy = False
# check if i have a remote units
remote_units = Annotation_Remote_Units_Annotation_Units.objects.all(
).filter(unit_id=au)
for ru in remote_units:
# retrieve its original unit
remote_original_unit = Annotation_Units.objects.get(
id=ru.remote_unit_id.id, task_id=obj.id)
# set the remote is_remote_copy = true
remote_original_unit.is_remote_copy = True
# set the parent_id to be the remote's one
remote_original_unit.parent_id = ru.unit_id
# add the remote original unit to the json output
annotation_units_json.append(
Annotation_UnitsSerializer(remote_original_unit).data)
annotation_units_json.append(Annotation_UnitsSerializer(au).data)
# return all array sorted with all the remote units in the end
return sorted(annotation_units_json,
key=operator.itemgetter('is_remote_copy'),
reverse=False)
# **********************************
# AS ROOT OBJECT
# **********************************
# try:
# au = Annotation_Units.objects.get(task_id_id=obj.id, parent_id=None)
# except Annotation_Units.DoesNotExist:
# au = None
# return Annotation_UnitsSerializer(au).data
def get_root_task(self, task_instance):
root_task = task_instance
while (root_task.parent_task != None):
root_task = root_task.parent_task
return root_task.id
class Meta:
model = Tasks
fields = ('id', 'parent', 'children', 'type', 'status', 'project',
'user', 'passage', 'tokens', 'annotation_units', 'is_demo',
'manager_comment', 'is_active', 'created_by', 'created_at',
'updated_at')
def update(self, instance, validated_data):
# disable saving a SUBMITTED task
if instance.status == 'SUBMITTED':
raise CantChangeSubmittedTaskExeption
save_type = self.initial_data['save_type']
if (save_type == 'draft'):
self.save_draft(instance)
elif (save_type == 'submit'):
self.submit(instance)
return instance
def save_draft(self, instance):
instance.status = 'ONGOING'
print('save_draft')
if (instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']):
self.save_tokenization_task(instance)
elif (instance.type == Constants.TASK_TYPES_JSON['ANNOTATION']):
self.save_annotation_task(instance)
elif (instance.type == Constants.TASK_TYPES_JSON['REVIEW']):
self.save_review_task(instance)
instance.save()
def save_tokenization_task(self, instance):
print('save_tokenization_task - start')
self.check_if_parent_task_ok_or_exception(instance)
instance.tokens_set.all().delete()
for token in self.initial_data['tokens']:
newToken = Tokens()
newToken.task_id_id = instance
newToken.text = token['text']
newToken.require_annotation = (not isPunct(newToken.text))
newToken.start_index = token['start_index']
newToken.end_index = token['end_index']
instance.tokens_set.add(newToken, bulk=False)
print('save_tokenization_task - end')
def save_annotation_task(self, instance):
print('save_annotation_task - start')
# mainly saving an annotations units array
self.check_if_parent_task_ok_or_exception(instance)
self.reset_current_task(instance)
remote_units_array = []
for au in self.initial_data['annotation_units']:
annotation_unit = Annotation_Units()
annotation_unit.annotation_unit_tree_id = au[
'annotation_unit_tree_id']
annotation_unit.task_id = instance
annotation_unit.type = au['type']
annotation_unit.comment = au['comment']
annotation_unit.is_remote_copy = au['is_remote_copy']
parent_id = None
if au['parent_id']:
parent_id = get_object_or_404(
Annotation_Units,
annotation_unit_tree_id=au['parent_id'],
task_id=instance.id)
annotation_unit.parent_id = parent_id
annotation_unit.gui_status = au['gui_status']
if annotation_unit.is_remote_copy == True:
annotation_unit.remote_categories = get_value_or_none(
'categories', au)
remote_units_array.append(annotation_unit)
else:
instance.annotation_units_set.add(annotation_unit, bulk=False)
self.save_children_tokens(
annotation_unit, get_value_or_none('children_tokens', au))
self.save_annotation_categories(
annotation_unit, get_value_or_none('categories', au))
for annotation_unit in remote_units_array:
remote_unit = self.save_annotation_remote_unit(annotation_unit)
self.save_remote_annotation_categories(
remote_unit, annotation_unit.remote_categories)
print('save_annotation_task - end')
def save_remote_annotation_categories(self, remote_annotation_unit,
categories):
print('save_remote_annotation_categories - start')
for cat in categories:
unit_category = Annotation_Units_Categories()
unit_category.unit_id = remote_annotation_unit.remote_unit_id
unit_category.category_id = Categories.objects.get(id=cat['id'])
unit_category.remote_parent_id = remote_annotation_unit.unit_id
unit_category.save()
print('save_remote_annotation_categories - end')
def reset_current_task(self, task_instance):
# TODO: validate the new array of annotation units before deleting the current one
print('reset_current_task - start')
# reset Annotation_Units_Tokens
# reset Annotation_Units_Categories
# reset Annotation_Remote_Units_Annotation_Units
# reset annotaion_units
task_instance.annotation_units_set.all().delete()
print('reset_current_task - end')
def save_annotation_remote_unit(self, annotation_unit):
remote_unit = Annotation_Remote_Units_Annotation_Units()
# remote_unit.unit_id means that it is the parent
remote_unit.unit_id = annotation_unit.parent_id
# remote_unit.remote_unit_id means that it is the remote unit
remote_unit_id = get_object_or_404(
Annotation_Units,
annotation_unit_tree_id=annotation_unit.annotation_unit_tree_id,
task_id=annotation_unit.task_id)
remote_unit.remote_unit_id = remote_unit_id
remote_unit.save()
return remote_unit
def save_children_tokens(self, annotation_unit, tokens):
if tokens != None:
print('save_children_tokens - start')
for t in tokens:
annotation_units_token = Annotation_Units_Tokens()
annotation_units_token.unit_id = annotation_unit
annotation_units_token.token_id = Tokens.objects.get(
id=t['id'])
annotation_units_token.save()
print('save_children_tokens - end')
def save_annotation_categories(self, annotation_unit, categories):
print('save_annotation_categories - start')
for cat in categories:
unit_category = Annotation_Units_Categories()
unit_category.unit_id = annotation_unit
unit_category.category_id = Categories.objects.get(id=cat['id'])
unit_category.remote_parent_id = None
unit_category.save()
print('save_annotation_categories - end')
def save_review_task(self, instance):
# TODO: CHECK IF OK !!!!
print('save_review_task - start')
self.save_annotation_task(instance)
print('save_review_task - end')
def submit(self, instance):
instance.status = 'SUBMITTED'
print('submit')
instance.save()
def check_if_parent_task_ok_or_exception(self, instance):
if instance.type == Constants.TASK_TYPES_JSON['TOKENIZATION']:
if instance.parent_task != None:
raise SaveTaskTypeDeniedException
elif instance.parent_task == None:
raise SaveTaskTypeDeniedException