def create_instance(instance_data, session):
instance = Instance(
project_id = instance_data.get('project_id'),
task_id = instance_data.get('task_id'),
type = instance_data.get('type'),
hash = instance_data.get('hash'),
status = instance_data.get('status'),
start_sentence = instance_data.get('start_sentence'),
end_sentence = instance_data.get('end_sentence'),
start_token = instance_data.get('start_token'),
end_token = instance_data.get('end_token'),
start_char = instance_data.get('start_char'),
end_char = instance_data.get('end_char'),
sentence = instance_data.get('sentence'),
sequence_id = instance_data.get('sequence_id'),
number = instance_data.get('number'),
frame_number = instance_data.get('frame_number'),
global_frame_number = instance_data.get('global_frame_number'),
machine_made = instance_data.get('machine_made'),
interpolated = instance_data.get('interpolated'),
fan_made = instance_data.get('fan_made'),
verified = instance_data.get('verified'),
occluded = instance_data.get('occluded'),
soft_delete = instance_data.get('soft_delete'),
label_file_id = instance_data.get('label_file_id'),
file_id = instance_data.get('file_id'),
points = instance_data.get('points'),
mask_url = instance_data.get('mask_url'),
mask_blob_dir = instance_data.get('mask_blob_dir'),
mask_url_expiry = instance_data.get('mask_url_expiry'),
x_min = instance_data.get('x_min'),
y_min = instance_data.get('y_min'),
x_max = instance_data.get('x_max'),
y_max = instance_data.get('y_max'),
width = instance_data.get('width'),
height = instance_data.get('height'),
preview_image_url = instance_data.get('preview_image_url'),
preview_image_blob_dir = instance_data.get('preview_image_blob_dir'),
preview_image_url_expiry = instance_data.get('preview_image_url_expiry'),
rating = instance_data.get('rating'),
rating_comment = instance_data.get('rating_comment'),
attribute_groups = instance_data.get('attribute_groups'),
member_created_id = instance_data.get('member_created_id'),
nodes = {'nodes': instance_data.get('nodes')},
edges = {'edges': instance_data.get('edges')},
previous_id = instance_data.get('previous_id'),
root_id = instance_data.get('root_id')
)
session.add(instance)
regular_methods.commit_with_rollback(session)
return instance
def regenerate_keyframe_list(self, session):
"""
"""
# Not yet tested.
# does not add to session
instance_list = Instance.list( # Using this checks for soft deleted by default
session=session,
sequence_id=self.id,
limit=None)
# print("len instance_list", len(instance_list))
for instance in instance_list:
self.update_frame_number_list(session, instance)
def test_get_child_instance_history(self):
root = data_mocking.create_instance({'previous_id': None},
self.session)
instance1 = data_mocking.create_instance({'root_id': root.id},
self.session)
instance2 = data_mocking.create_instance({'root_id': root.id},
self.session)
instance3 = data_mocking.create_instance({'root_id': root.id},
self.session)
instance4 = data_mocking.create_instance({'root_id': 0}, self.session)
history = Instance.get_child_instance_history(self.session, root.id)
self.assertEqual(len(history), 4)
self.assertTrue(instance1 in history)
self.assertTrue(instance2 in history)
self.assertTrue(instance3 in history)
self.assertTrue(root in history)
self.assertTrue(instance4 not in history)
def copy_file_from_existing(session,
working_dir,
existing_file,
copy_instance_list: bool = False,
log=regular_log.default(),
add_link: bool = True,
remove_link: bool = True,
orginal_directory_id=None,
previous_video_parent_id=None,
sequence_map=None,
deep_copy=False,
defer_copy=False,
ann_is_complete_reset=False,
batch_id=None,
flush_session=False,
working_dir_id: int = None):
"""
orginal_directory_id is for Video, to get list of video files
Should we rename to "source_directory" to keep in line of transfer thing?
Clarify working_dir is the "target" directory?
Don't actually need directory if not copying links
# TODO is "update" really the right name
if this is generally creating a new file??
If file is video, we need to
* Create the new video file
* Create new files for all of it's frames
ann_is_complete_reset
We assume "copying" means copying the status too.
However, for the example of tasks (and perhaps others in future)
we assume we want to reset this status for the newly created files.
"""
# Defer image copy is specified in the parameter.
start_time = time.time()
if working_dir:
working_dir_id = working_dir.id
# IMAGE Defer
if existing_file.type == 'image' and defer_copy and not remove_link:
regular_methods.transmit_interservice_request_after_commit(
session=session,
message='image_copy',
logger=logger,
service_target='walrus',
id=existing_file.id,
project_string_id=existing_file.project.project_string_id,
extra_params={
'file_id': existing_file.id,
'copy_instance_list': copy_instance_list,
'destination_working_dir_id': working_dir_id,
'source_working_dir_id': orginal_directory_id,
'add_link': add_link,
'batch_id': batch_id,
'remove_link': remove_link,
})
log['info'][
'message'] = 'File copy in progress. Please check progress in the file operations progress section.'
return
# VIDEO
if existing_file.type == "video" and defer_copy is True:
# Defer the copy to the walrus.
regular_methods.transmit_interservice_request_after_commit(
session=session,
message='video_copy',
logger=logger,
service_target='walrus',
id=existing_file.id,
project_string_id=existing_file.project.project_string_id,
extra_params={
'file_id': existing_file.id,
'copy_instance_list': copy_instance_list,
'destination_working_dir_id': working_dir_id,
'source_working_dir_id': orginal_directory_id,
'add_link': add_link,
'batch_id': batch_id,
'remove_link': remove_link,
'frame_count': existing_file.video.frame_count
})
log['info'][
'message'] = 'File copy in progress. Please check progress in the file operations progress section.'
return
file = new_file_database_object_from_existing(session)
file.type = existing_file.type
# We need this to do permissions
# At the moment when a video is done
# We only move the video into the directory not the images
# So we need to use the project scope.
file.project_id = existing_file.project_id
file.image_id = existing_file.image_id
file.label_id = existing_file.label_id
file.video_id = existing_file.video_id
file.global_frame_number = existing_file.global_frame_number
file.colour = existing_file.colour
file_relationship(session, file, existing_file)
file.state = "changed"
file.frame_number = existing_file.frame_number # Ok if None
if ann_is_complete_reset is False:
file.ann_is_complete = existing_file.ann_is_complete
file.original_filename = existing_file.original_filename
# Want to be able to get video file from anyframe...
# Careful this is the previous one, if we just copy existing then images will
# be related to old file
file.video_parent_file_id = previous_video_parent_id
session.add(file)
# Question why does add link ned to be true here? or does it?
# At the moment we don't pass add_link as True when copying it for task
if add_link is True:
working_dir_database_models.WorkingDirFileLink.add(
session, working_dir_id, file)
# print("Added link")
if remove_link is True:
working_dir_database_models.WorkingDirFileLink.remove(
session, working_dir_id, existing_file.id)
logger.debug('existing_file.type {}'.format(existing_file.type))
logger.debug('copy_instance_list {}'.format(copy_instance_list))
if existing_file.type in ['image', 'frame'
] and copy_instance_list is True:
file.count_instances_changed = existing_file.count_instances_changed
file.set_cache_key_dirty('instance_list')
instance_list = Instance.list(
session=session, file_id=existing_file.id,
limit=None) # Excludes removed by default
logger.debug('instance_list len {}'.format(len(instance_list)))
for instance in instance_list:
instance_sequence_id = instance.sequence_id
if sequence_map is not None:
logger.debug('sequence_map {}'.format(sequence_map))
instance_sequence_id = sequence_map.get(
instance_sequence_id)
new_instance = Instance(
file_id=file.id, # IMPORTANT and different from pattern
sequence_id=instance_sequence_id, # Different
parent_file_id=file.
video_parent_file_id, # Cache for video parent file ID.
project_id=instance.project_id,
x_min=instance.x_min,
y_min=instance.y_min,
x_max=instance.x_max,
y_max=instance.y_max,
width=instance.width,
height=instance.height,
label_file_id=instance.label_file_id,
hash=instance.hash,
type=instance.type,
number=instance.number,
frame_number=instance.frame_number,
global_frame_number=instance.global_frame_number,
machine_made=instance.machine_made,
fan_made=instance.fan_made,
points=instance.points,
soft_delete=instance.soft_delete,
center_x=instance.center_x,
center_y=instance.center_y,
angle=instance.angle,
p1=instance.p1,
p2=instance.p2,
cp=instance.cp,
interpolated=instance.interpolated,
front_face=instance.front_face,
rear_face=instance.rear_face,
creation_ref_id=instance.creation_ref_id)
session.add(new_instance)
end_time = time.time()
if flush_session:
session.flush()
return file
def update_frame_number_list(self, session, instance):
# Use instance for frame number and flag...
frame_number_list = self.keyframe_list['frame_number_list']
default_sequences_to_single_frame = instance.label_file.label.default_sequences_to_single_frame
if instance.soft_delete is False or instance.soft_delete is None:
# there wasn't a super obvious way to use built in
# bisect for this test and since we expect this list to
# generally be < 100 elements just leaving it.
# this (not in) test is here because it was doubling up
# keyframes, somethign to do with the way attributes were
# saving maybe? but it was suprisingly glaring bug
# on front end
if instance.frame_number not in frame_number_list:
# binary search based insertion
bisect.insort(frame_number_list, instance.frame_number)
elif instance.soft_delete and not default_sequences_to_single_frame:
try:
del frame_number_list[bisect.bisect_left(
frame_number_list, instance.frame_number)]
except:
pass
# NOTE we previously had concept of if len(frame_number_list) == 0:
# Which could then call delete_sequence_shared()
# This was not fully implmented so leaving for now, but this is the "hook"
# where we would put it (at least in prior knowledge conextex.)
elif instance.soft_delete and default_sequences_to_single_frame:
# Opportunity to have some sort of cache to avoid querying here since this can sometimes be called
# Inside a for loop.
"""
1) Future idea. All of this info should be available on
`instance_list_kept_serialized`... because, generally speaking
we have all the instances already so shouldn't have to do a second
query.
2) There could be a timing issue here, we seem to do this update
"opportunistically", eg as each instance completes ...
but what if more then one change in the one save event
"""
count_instances_on_same_frame_and_sequence = Instance.list(
session=session,
sequence_id=instance.sequence_id,
file_id=instance.file_id,
frame_number=instance.frame_number,
label_file_id=instance.label_file_id,
limit=None,
return_kind="count")
# logger.info(count_instances_on_same_frame_and_sequence)
if count_instances_on_same_frame_and_sequence == 1:
# Delete frame number list if its the last instance on this keyframe from same seq #.
try:
del frame_number_list[bisect.bisect_left(
frame_number_list, instance.frame_number)]
except:
pass
self.keyframe_list['frame_number_list'] = frame_number_list