def main():
create_results_dir_in_s3(experiment_name=EXPERIMENT_NAME)
# variables: initial_model_idx, curr_thresh
original_train_annotations = os.path.join(
ANNOTATIONS_DIR, NEW_ANNOTATIONS_DIR,
'{prefix}_{model_idx}'.format(prefix=NEW_ANNOTATIONS_FILE_PREFIX,
model_idx=0))
next_gen_annotations = os.path.join(
ANNOTATIONS_DIR, NEW_ANNOTATIONS_DIR,
'{prefix}_{model_idx}'.format(prefix=NEW_ANNOTATIONS_FILE_PREFIX,
model_idx=1))
if INITIAL_MODEL is not None:
if INITIAL_MODEL_BUCKET is not None:
# assuming model name is in format model_type_{model_type}_model_no_{model_idx}
initial_model_idx = int(INITIAL_MODEL[-1])
curr_thresh = ANNOTATIONS_SCORE_INITIAL_THRESH
# download model from S3 bucket
s3 = boto3.resource('s3',
aws_access_key_id=AWS_ACCESS_ID,
aws_secret_access_key=AWS_ACCESS_KEY)
local_model_path = os.path.join(OPENPIFPAF_PATH, INITIAL_MODEL)
s3.meta.client.download_file(INITIAL_MODEL_BUCKET, INITIAL_MODEL,
local_model_path)
logging.info(
'********************************************************************'
)
logging.info(
'************************* Model No {model_idx}. *************************'
.format(model_idx=initial_model_idx))
logging.info(
'********************************************************************'
)
teacher = Teacher(
model_type='openpifpaf',
model_idx=initial_model_idx,
num_train_epochs=NUM_TRAIN_EPOCHS,
train_image_dir=TRAIN_IMAGE_DIR,
train_annotations=original_train_annotations,
original_train_annotations=original_train_annotations,
val_image_dir=VAL_IMAGE_DIR,
val_annotations=os.path.join(ANNOTATIONS_DIR,
ORIGINAL_ANNOTATIONS_DIR,
ORIGINAL_VAL_ANNOTATION_FILE),
next_gen_annotations=next_gen_annotations)
else:
raise EnvironmentError(
'INITIAL_MODEL_BUCKET should exist if INITIAL_MODEL exists')
else:
initial_model_idx = 0
curr_thresh = ANNOTATIONS_SCORE_INITIAL_THRESH
logging.info(
'********************************************************************'
)
logging.info(
'************************* Model No {model_idx}. *************************'
.format(model_idx=initial_model_idx))
logging.info(
'********************************************************************'
)
teacher = Teacher(
model_type='openpifpaf',
model_idx=initial_model_idx,
num_train_epochs=NUM_TRAIN_EPOCHS,
train_image_dir=TRAIN_IMAGE_DIR,
train_annotations=original_train_annotations,
original_train_annotations=original_train_annotations,
val_image_dir=VAL_IMAGE_DIR,
val_annotations=os.path.join(ANNOTATIONS_DIR,
ORIGINAL_ANNOTATIONS_DIR,
ORIGINAL_VAL_ANNOTATION_FILE),
next_gen_annotations=next_gen_annotations)
logging.info(
'Fitting Model no.{model_idx}'.format(model_idx=initial_model_idx))
teacher.fit()
logging.info('Creating Validation Scores to Model no.{model_idx}'.format(
model_idx=initial_model_idx))
teacher.create_val_score()
logging.info(
'Creating New data Scores and New Annotations to Model no.{model_idx}'.
format(model_idx=initial_model_idx))
teacher.create_new_data_scores_and_annotations(thresh=curr_thresh)
teacher.save_annotations(experiment_name=EXPERIMENT_NAME)
curr_thresh = round(max(0.2, curr_thresh - ANNOTATION_SCORE_DECREASE), 3)
teacher.save_results(experiment_name=EXPERIMENT_NAME)
teacher.save_logs(experiment_name=EXPERIMENT_NAME)
teacher.save_model(experiment_name=EXPERIMENT_NAME)
if MOCK_ONE_MODEL == 'TRUE':
return
for model_idx in range(initial_model_idx + 1, STUDENT_TEACHER_LOOP):
last_model_in_loop = model_idx == STUDENT_TEACHER_LOOP - 1
if not last_model_in_loop:
curr_next_gen_annotations = next_gen_annotations
else:
curr_next_gen_annotations = None
new_student = Student(
model_type='openpifpaf',
model_idx=model_idx,
num_train_epochs=NUM_TRAIN_EPOCHS,
train_image_dir=TRAIN_IMAGE_DIR,
train_annotations=os.path.join(
OPENPIFPAF_PATH, '{prefix}_{model_idx}'.format(
prefix=MERGED_TRAIN_ANNOTATIONS_FILE_PREFIX,
model_idx=model_idx)),
original_train_annotations=original_train_annotations,
val_image_dir=VAL_IMAGE_DIR,
val_annotations=os.path.join(ANNOTATIONS_DIR,
ORIGINAL_ANNOTATIONS_DIR,
ORIGINAL_VAL_ANNOTATION_FILE),
next_gen_annotations=curr_next_gen_annotations)
logging.info(
'********************************************************************'
)
logging.info(
'************************* Model No {model_idx}. *************************'
.format(model_idx=model_idx))
logging.info(
'********************************************************************'
)
logging.info(
'Fitting Model no.{model_idx}'.format(model_idx=model_idx))
new_student.fit()
teacher = new_student
logging.info(
'Creating Validation Scores to Model no.{model_idx}'.format(
model_idx=model_idx))
teacher.create_val_score()
if not last_model_in_loop:
logging.info(
'Creating New data Scores and New Annotations to Model no.{model_idx}'
.format(model_idx=model_idx))
teacher.create_new_data_scores_and_annotations(thresh=curr_thresh)
teacher.save_annotations(experiment_name=EXPERIMENT_NAME)
curr_thresh = round(
max(0.2, curr_thresh - ANNOTATION_SCORE_DECREASE), 3)
teacher.save_results(experiment_name=EXPERIMENT_NAME)
teacher.save_logs(experiment_name=EXPERIMENT_NAME)
teacher.save_model(experiment_name=EXPERIMENT_NAME)
if RUN_FULL_MODEL == 'TRUE':
full_data_model = create_full_data_model_for_comparison(model_idx + 1)
full_data_model.fit()
full_data_model.create_val_score()
full_data_model.save_results(experiment_name=EXPERIMENT_NAME)
full_data_model.save_logs(experiment_name=EXPERIMENT_NAME)
full_data_model.save_model(experiment_name=EXPERIMENT_NAME)
upload_tb_logs_to_s3(experiment_name=EXPERIMENT_NAME)
def main():
create_results_dir_in_s3(experiment_name=EXPERIMENT_NAME)
# variables: initial_model_idx, curr_thresh
initial_model_idx = 0
curr_thresh = ANNOTATIONS_SCORE_INITIAL_THRESH
original_train_annotations = os.path.join(
ANNOTATIONS_DIR, NEW_ANNOTATIONS_DIR,
'{prefix}_{model_idx}'.format(prefix=NEW_ANNOTATIONS_FILE_PREFIX,
model_idx=initial_model_idx))
next_gen_annotations = os.path.join(
ANNOTATIONS_DIR, NEW_ANNOTATIONS_DIR,
'{prefix}_{model_idx}'.format(prefix=NEW_ANNOTATIONS_FILE_PREFIX,
model_idx=initial_model_idx + 1))
teacher = Teacher(model_type='openpifpaf',
model_idx=initial_model_idx,
num_train_epochs=NUM_TRAIN_EPOCHS,
train_image_dir=TRAIN_IMAGE_DIR,
train_annotations=original_train_annotations,
original_train_annotations=original_train_annotations,
val_image_dir=VAL_IMAGE_DIR,
val_annotations=os.path.join(
ANNOTATIONS_DIR, ORIGINAL_ANNOTATIONS_DIR,
ORIGINAL_VAL_ANNOTATION_FILE),
next_gen_annotations=next_gen_annotations)
logging.info(
'********************************************************************')
logging.info(
'************************* Model No {model_idx}. *************************'
.format(model_idx=initial_model_idx))
logging.info(
'********************************************************************')
logging.info(
'Fitting Model no.{model_idx}'.format(model_idx=initial_model_idx))
teacher.fit()
logging.info('Creating Validation Scores to Model no.{model_idx}'.format(
model_idx=initial_model_idx))
teacher.create_val_score()
logging.info(
'Creating New data Scores and New Annotations to Model no.{model_idx}'.
format(model_idx=initial_model_idx))
teacher.create_new_data_scores_and_annotations(thresh=curr_thresh)
teacher.save_annotations(experiment_name=EXPERIMENT_NAME)
curr_thresh = round(max(0.2, curr_thresh - ANNOTATION_SCORE_DECREASE), 3)
teacher.save_results(experiment_name=EXPERIMENT_NAME)
teacher.save_logs(experiment_name=EXPERIMENT_NAME)
teacher.save_model(experiment_name=EXPERIMENT_NAME)
if MOCK_ONE_MODEL == 'TRUE':
return
for model_idx in range(initial_model_idx + 1, STUDENT_TEACHER_LOOP):
last_model_in_loop = model_idx == STUDENT_TEACHER_LOOP - 1
if not last_model_in_loop:
curr_next_gen_annotations = next_gen_annotations
else:
curr_next_gen_annotations = None
new_student = Student(
model_type='openpifpaf',
model_idx=model_idx,
num_train_epochs=NUM_TRAIN_EPOCHS,
train_image_dir=TRAIN_IMAGE_DIR,
train_annotations=os.path.join(
OPENPIFPAF_PATH, '{prefix}_{model_idx}'.format(
prefix=MERGED_TRAIN_ANNOTATIONS_FILE_PREFIX,
model_idx=model_idx)),
original_train_annotations=original_train_annotations,
val_image_dir=VAL_IMAGE_DIR,
val_annotations=os.path.join(ANNOTATIONS_DIR,
ORIGINAL_ANNOTATIONS_DIR,
ORIGINAL_VAL_ANNOTATION_FILE),
next_gen_annotations=curr_next_gen_annotations)
logging.info(
'********************************************************************'
)
logging.info(
'************************* Model No {model_idx}. *************************'
.format(model_idx=model_idx))
logging.info(
'********************************************************************'
)
logging.info(
'Fitting Model no.{model_idx}'.format(model_idx=model_idx))
new_student.fit()
teacher = new_student
logging.info(
'Creating Validation Scores to Model no.{model_idx}'.format(
model_idx=model_idx))
teacher.create_val_score()
if not last_model_in_loop:
logging.info(
'Creating New data Scores and New Annotations to Model no.{model_idx}'
.format(model_idx=model_idx))
teacher.create_new_data_scores_and_annotations(thresh=curr_thresh)
teacher.save_annotations(experiment_name=EXPERIMENT_NAME)
curr_thresh = round(
max(0.2, curr_thresh - ANNOTATION_SCORE_DECREASE), 3)
teacher.save_results(experiment_name=EXPERIMENT_NAME)
teacher.save_logs(experiment_name=EXPERIMENT_NAME)
teacher.save_model(experiment_name=EXPERIMENT_NAME)
if RUN_FULL_MODEL == 'TRUE':
full_data_model = create_full_data_model_for_comparison(model_idx + 1)
full_data_model.fit()
full_data_model.create_val_score()
full_data_model.save_results(experiment_name=EXPERIMENT_NAME)
full_data_model.save_logs(experiment_name=EXPERIMENT_NAME)
full_data_model.save_model(experiment_name=EXPERIMENT_NAME)
upload_tb_logs_to_s3(experiment_name=EXPERIMENT_NAME)