def make_action_parameters(team_uuid, dataset_uuid, dataset_zip_uuid, partition_count):
action_parameters = action.create_action_parameters(action.ACTION_NAME_DATASET_ZIP)
action_parameters['team_uuid'] = team_uuid
action_parameters['dataset_uuid'] = dataset_uuid
action_parameters['dataset_zip_uuid'] = dataset_zip_uuid
action_parameters['partition_count'] = partition_count
return action_parameters
def make_action_parameters(team_uuid, dataset_uuid, video_uuids_json, eval_percent, create_time_ms):
action_parameters = action.create_action_parameters(action.ACTION_NAME_DATASET_PRODUCE)
action_parameters['team_uuid'] = team_uuid
action_parameters['dataset_uuid'] = dataset_uuid
action_parameters['video_uuids_json'] = video_uuids_json
action_parameters['eval_percent'] = eval_percent
action_parameters['create_time_ms'] = create_time_ms
return action_parameters
def start_frame_extraction(team_uuid, video_uuid):
storage.prepare_to_start_frame_extraction(team_uuid, video_uuid)
action_parameters = action.create_action_parameters(
action.ACTION_NAME_FRAME_EXTRACTION)
action_parameters['team_uuid'] = team_uuid
action_parameters['video_uuid'] = video_uuid
action.trigger_action_via_blob(action_parameters)
return action_parameters
def prepare_to_start_tracking(team_uuid, video_uuid, tracker_name, scale,
init_frame_number, init_bboxes_text):
tracker_uuid = storage.tracker_starting(team_uuid, video_uuid,
tracker_name, scale,
init_frame_number,
init_bboxes_text)
action_parameters = action.create_action_parameters(
action.ACTION_NAME_TRACKING)
action_parameters['video_uuid'] = video_uuid
action_parameters['tracker_uuid'] = tracker_uuid
action.trigger_action_via_blob(action_parameters)
return tracker_uuid
def zip_dataset(action_parameters):
team_uuid = action_parameters['team_uuid']
dataset_zip_uuid = action_parameters['dataset_zip_uuid']
dataset_uuid = action_parameters['dataset_uuid']
partition_count = action_parameters['partition_count']
partition_lists = [[] for i in range(partition_count)]
dataset_entity = storage.retrieve_dataset_entity(team_uuid, dataset_uuid)
partition_lists[0].append(dataset_entity['label_map_blob_name'])
dataset_record_entities = storage.retrieve_dataset_records(dataset_entity)
for i, dataset_record_entity in enumerate(dataset_record_entities):
partition_lists[(i + 1) % partition_count].append(dataset_record_entity['tf_record_blob_name'])
# Trigger actions for the partitions
action_parameters = action.create_action_parameters(action.ACTION_NAME_DATASET_ZIP_PARTITION)
action_parameters['team_uuid'] = team_uuid
action_parameters['dataset_zip_uuid'] = dataset_zip_uuid
for partition_index, partition_list in enumerate(partition_lists):
file_count = len(partition_list)
storage.update_dataset_zipper(team_uuid, dataset_zip_uuid, partition_index, file_count, 0)
action_parameters['partition_list'] = partition_list
action_parameters['partition_index'] = partition_index
action.trigger_action_via_blob(action_parameters)
def make_action_parameters(team_uuid, model_uuid):
action_parameters = action.create_action_parameters(
action.ACTION_NAME_EXTRACT_SUMMARY_IMAGES)
action_parameters['team_uuid'] = team_uuid
action_parameters['model_uuid'] = model_uuid
return action_parameters
def produce_dataset(action_parameters):
team_uuid = action_parameters['team_uuid']
dataset_uuid = action_parameters['dataset_uuid']
video_uuids_json = action_parameters['video_uuids_json']
eval_percent = action_parameters['eval_percent']
create_time_ms = action_parameters['create_time_ms']
video_uuid_list = json.loads(video_uuids_json)
if len(video_uuid_list) == 0:
message = "Error: No videos to process."
logging.critical(message)
raise exceptions.HttpErrorBadRequest(message)
video_entities = storage.retrieve_video_entities(team_uuid,
video_uuid_list)
if len(video_entities) != len(video_uuid_list):
message = 'Error: One or more videos not found for video_uuids=%s.' % video_uuids_json
logging.critical(message)
raise exceptions.HttpErrorNotFound(message)
dict_video_uuid_to_split = {}
train_frame_count = 0
train_record_count = 0
eval_frame_count = 0
eval_record_count = 0
label_set = set()
for video_entity in video_entities:
video_uuid = video_entity['video_uuid']
# Read the video_frame entities from storage. They contain the labels.
video_frame_entities = storage.retrieve_video_frame_entities(
team_uuid, video_uuid, 0, video_entity['frame_count'] - 1)
# Determine which frames will be used for training and which frames will be used for eval.
split = __split_for_records(video_frame_entities, eval_percent)
dict_video_uuid_to_split[video_uuid] = split
train_frame_count += split.train_frame_count
train_record_count += len(split.train_frame_number_lists)
eval_frame_count += split.eval_frame_count
eval_record_count += len(split.eval_frame_number_lists)
label_set.update(split.label_set)
sorted_label_list = sorted(label_set)
train_record_id_format = 'train_dataset.record-%05d-%05d'
train_input_path = 'train_dataset.record-?????-%05d' % train_record_count
eval_record_id_format = 'eval_dataset.record-%05d-%05d'
eval_input_path = 'eval_dataset.record-?????-%05d' % eval_record_count
storage.dataset_producer_starting(team_uuid, dataset_uuid,
sorted_label_list, train_frame_count,
train_record_count, train_input_path,
eval_frame_count, eval_record_count,
eval_input_path)
record_number = 0
train_record_number = 0
eval_record_number = 0
action_parameters = action.create_action_parameters(
action.ACTION_NAME_DATASET_PRODUCE_RECORD)
action_parameters['team_uuid'] = team_uuid
action_parameters['dataset_uuid'] = dataset_uuid
action_parameters['sorted_label_list'] = sorted_label_list
# Trigger actions for the train records
for video_entity in video_entities:
video_uuid = video_entity['video_uuid']
split = dict_video_uuid_to_split[video_uuid]
action_parameters['video_uuid'] = video_uuid
for i, train_frame_number_list in enumerate(
split.train_frame_number_lists):
action_parameters['frame_number_list'] = train_frame_number_list
action_parameters['record_number'] = record_number
action_parameters['record_id'] = train_record_id_format % (
train_record_number, train_record_count)
action_parameters['is_eval'] = False
action.trigger_action_via_blob(action_parameters)
train_record_number += 1
record_number += 1
# Trigger actions for the eval records
for video_entity in video_entities:
video_uuid = video_entity['video_uuid']
split = dict_video_uuid_to_split[video_uuid]
action_parameters['video_uuid'] = video_uuid
for i, eval_frame_number_list in enumerate(
split.eval_frame_number_lists):
action_parameters['frame_number_list'] = eval_frame_number_list
action_parameters['record_number'] = record_number
action_parameters['record_id'] = eval_record_id_format % (
eval_record_number, eval_record_count)
action_parameters['is_eval'] = True
action.trigger_action_via_blob(action_parameters)
eval_record_number += 1
record_number += 1