def main(): create_results_dir_in_s3(experiment_name=EXPERIMENT_NAME) # variables: initial_model_idx, curr_thresh original_train_annotations = os.path.join( ANNOTATIONS_DIR, NEW_ANNOTATIONS_DIR, '{prefix}_{model_idx}'.format(prefix=NEW_ANNOTATIONS_FILE_PREFIX, model_idx=0)) next_gen_annotations = os.path.join( ANNOTATIONS_DIR, NEW_ANNOTATIONS_DIR, '{prefix}_{model_idx}'.format(prefix=NEW_ANNOTATIONS_FILE_PREFIX, model_idx=1)) if INITIAL_MODEL is not None: if INITIAL_MODEL_BUCKET is not None: # assuming model name is in format model_type_{model_type}_model_no_{model_idx} initial_model_idx = int(INITIAL_MODEL[-1]) curr_thresh = ANNOTATIONS_SCORE_INITIAL_THRESH # download model from S3 bucket s3 = boto3.resource('s3', aws_access_key_id=AWS_ACCESS_ID, aws_secret_access_key=AWS_ACCESS_KEY) local_model_path = os.path.join(OPENPIFPAF_PATH, INITIAL_MODEL) s3.meta.client.download_file(INITIAL_MODEL_BUCKET, INITIAL_MODEL, local_model_path) logging.info( '********************************************************************' ) logging.info( '************************* Model No {model_idx}. *************************' .format(model_idx=initial_model_idx)) logging.info( '********************************************************************' ) teacher = Teacher( model_type='openpifpaf', model_idx=initial_model_idx, num_train_epochs=NUM_TRAIN_EPOCHS, train_image_dir=TRAIN_IMAGE_DIR, train_annotations=original_train_annotations, original_train_annotations=original_train_annotations, val_image_dir=VAL_IMAGE_DIR, val_annotations=os.path.join(ANNOTATIONS_DIR, ORIGINAL_ANNOTATIONS_DIR, ORIGINAL_VAL_ANNOTATION_FILE), next_gen_annotations=next_gen_annotations) else: raise EnvironmentError( 'INITIAL_MODEL_BUCKET should exist if INITIAL_MODEL exists') else: initial_model_idx = 0 curr_thresh = ANNOTATIONS_SCORE_INITIAL_THRESH logging.info( '********************************************************************' ) logging.info( '************************* Model No {model_idx}. *************************' .format(model_idx=initial_model_idx)) logging.info( '********************************************************************' ) teacher = Teacher( model_type='openpifpaf', model_idx=initial_model_idx, num_train_epochs=NUM_TRAIN_EPOCHS, train_image_dir=TRAIN_IMAGE_DIR, train_annotations=original_train_annotations, original_train_annotations=original_train_annotations, val_image_dir=VAL_IMAGE_DIR, val_annotations=os.path.join(ANNOTATIONS_DIR, ORIGINAL_ANNOTATIONS_DIR, ORIGINAL_VAL_ANNOTATION_FILE), next_gen_annotations=next_gen_annotations) logging.info( 'Fitting Model no.{model_idx}'.format(model_idx=initial_model_idx)) teacher.fit() logging.info('Creating Validation Scores to Model no.{model_idx}'.format( model_idx=initial_model_idx)) teacher.create_val_score() logging.info( 'Creating New data Scores and New Annotations to Model no.{model_idx}'. format(model_idx=initial_model_idx)) teacher.create_new_data_scores_and_annotations(thresh=curr_thresh) teacher.save_annotations(experiment_name=EXPERIMENT_NAME) curr_thresh = round(max(0.2, curr_thresh - ANNOTATION_SCORE_DECREASE), 3) teacher.save_results(experiment_name=EXPERIMENT_NAME) teacher.save_logs(experiment_name=EXPERIMENT_NAME) teacher.save_model(experiment_name=EXPERIMENT_NAME) if MOCK_ONE_MODEL == 'TRUE': return for model_idx in range(initial_model_idx + 1, STUDENT_TEACHER_LOOP): last_model_in_loop = model_idx == STUDENT_TEACHER_LOOP - 1 if not last_model_in_loop: curr_next_gen_annotations = next_gen_annotations else: curr_next_gen_annotations = None new_student = Student( model_type='openpifpaf', model_idx=model_idx, num_train_epochs=NUM_TRAIN_EPOCHS, train_image_dir=TRAIN_IMAGE_DIR, train_annotations=os.path.join( OPENPIFPAF_PATH, '{prefix}_{model_idx}'.format( prefix=MERGED_TRAIN_ANNOTATIONS_FILE_PREFIX, model_idx=model_idx)), original_train_annotations=original_train_annotations, val_image_dir=VAL_IMAGE_DIR, val_annotations=os.path.join(ANNOTATIONS_DIR, ORIGINAL_ANNOTATIONS_DIR, ORIGINAL_VAL_ANNOTATION_FILE), next_gen_annotations=curr_next_gen_annotations) logging.info( '********************************************************************' ) logging.info( '************************* Model No {model_idx}. *************************' .format(model_idx=model_idx)) logging.info( '********************************************************************' ) logging.info( 'Fitting Model no.{model_idx}'.format(model_idx=model_idx)) new_student.fit() teacher = new_student logging.info( 'Creating Validation Scores to Model no.{model_idx}'.format( model_idx=model_idx)) teacher.create_val_score() if not last_model_in_loop: logging.info( 'Creating New data Scores and New Annotations to Model no.{model_idx}' .format(model_idx=model_idx)) teacher.create_new_data_scores_and_annotations(thresh=curr_thresh) teacher.save_annotations(experiment_name=EXPERIMENT_NAME) curr_thresh = round( max(0.2, curr_thresh - ANNOTATION_SCORE_DECREASE), 3) teacher.save_results(experiment_name=EXPERIMENT_NAME) teacher.save_logs(experiment_name=EXPERIMENT_NAME) teacher.save_model(experiment_name=EXPERIMENT_NAME) if RUN_FULL_MODEL == 'TRUE': full_data_model = create_full_data_model_for_comparison(model_idx + 1) full_data_model.fit() full_data_model.create_val_score() full_data_model.save_results(experiment_name=EXPERIMENT_NAME) full_data_model.save_logs(experiment_name=EXPERIMENT_NAME) full_data_model.save_model(experiment_name=EXPERIMENT_NAME) upload_tb_logs_to_s3(experiment_name=EXPERIMENT_NAME)
def main(): create_results_dir_in_s3(experiment_name=EXPERIMENT_NAME) # variables: initial_model_idx, curr_thresh initial_model_idx = 0 curr_thresh = ANNOTATIONS_SCORE_INITIAL_THRESH original_train_annotations = os.path.join( ANNOTATIONS_DIR, NEW_ANNOTATIONS_DIR, '{prefix}_{model_idx}'.format(prefix=NEW_ANNOTATIONS_FILE_PREFIX, model_idx=initial_model_idx)) next_gen_annotations = os.path.join( ANNOTATIONS_DIR, NEW_ANNOTATIONS_DIR, '{prefix}_{model_idx}'.format(prefix=NEW_ANNOTATIONS_FILE_PREFIX, model_idx=initial_model_idx + 1)) teacher = Teacher(model_type='openpifpaf', model_idx=initial_model_idx, num_train_epochs=NUM_TRAIN_EPOCHS, train_image_dir=TRAIN_IMAGE_DIR, train_annotations=original_train_annotations, original_train_annotations=original_train_annotations, val_image_dir=VAL_IMAGE_DIR, val_annotations=os.path.join( ANNOTATIONS_DIR, ORIGINAL_ANNOTATIONS_DIR, ORIGINAL_VAL_ANNOTATION_FILE), next_gen_annotations=next_gen_annotations) logging.info( '********************************************************************') logging.info( '************************* Model No {model_idx}. *************************' .format(model_idx=initial_model_idx)) logging.info( '********************************************************************') logging.info( 'Fitting Model no.{model_idx}'.format(model_idx=initial_model_idx)) teacher.fit() logging.info('Creating Validation Scores to Model no.{model_idx}'.format( model_idx=initial_model_idx)) teacher.create_val_score() logging.info( 'Creating New data Scores and New Annotations to Model no.{model_idx}'. format(model_idx=initial_model_idx)) teacher.create_new_data_scores_and_annotations(thresh=curr_thresh) teacher.save_annotations(experiment_name=EXPERIMENT_NAME) curr_thresh = round(max(0.2, curr_thresh - ANNOTATION_SCORE_DECREASE), 3) teacher.save_results(experiment_name=EXPERIMENT_NAME) teacher.save_logs(experiment_name=EXPERIMENT_NAME) teacher.save_model(experiment_name=EXPERIMENT_NAME) if MOCK_ONE_MODEL == 'TRUE': return for model_idx in range(initial_model_idx + 1, STUDENT_TEACHER_LOOP): last_model_in_loop = model_idx == STUDENT_TEACHER_LOOP - 1 if not last_model_in_loop: curr_next_gen_annotations = next_gen_annotations else: curr_next_gen_annotations = None new_student = Student( model_type='openpifpaf', model_idx=model_idx, num_train_epochs=NUM_TRAIN_EPOCHS, train_image_dir=TRAIN_IMAGE_DIR, train_annotations=os.path.join( OPENPIFPAF_PATH, '{prefix}_{model_idx}'.format( prefix=MERGED_TRAIN_ANNOTATIONS_FILE_PREFIX, model_idx=model_idx)), original_train_annotations=original_train_annotations, val_image_dir=VAL_IMAGE_DIR, val_annotations=os.path.join(ANNOTATIONS_DIR, ORIGINAL_ANNOTATIONS_DIR, ORIGINAL_VAL_ANNOTATION_FILE), next_gen_annotations=curr_next_gen_annotations) logging.info( '********************************************************************' ) logging.info( '************************* Model No {model_idx}. *************************' .format(model_idx=model_idx)) logging.info( '********************************************************************' ) logging.info( 'Fitting Model no.{model_idx}'.format(model_idx=model_idx)) new_student.fit() teacher = new_student logging.info( 'Creating Validation Scores to Model no.{model_idx}'.format( model_idx=model_idx)) teacher.create_val_score() if not last_model_in_loop: logging.info( 'Creating New data Scores and New Annotations to Model no.{model_idx}' .format(model_idx=model_idx)) teacher.create_new_data_scores_and_annotations(thresh=curr_thresh) teacher.save_annotations(experiment_name=EXPERIMENT_NAME) curr_thresh = round( max(0.2, curr_thresh - ANNOTATION_SCORE_DECREASE), 3) teacher.save_results(experiment_name=EXPERIMENT_NAME) teacher.save_logs(experiment_name=EXPERIMENT_NAME) teacher.save_model(experiment_name=EXPERIMENT_NAME) if RUN_FULL_MODEL == 'TRUE': full_data_model = create_full_data_model_for_comparison(model_idx + 1) full_data_model.fit() full_data_model.create_val_score() full_data_model.save_results(experiment_name=EXPERIMENT_NAME) full_data_model.save_logs(experiment_name=EXPERIMENT_NAME) full_data_model.save_model(experiment_name=EXPERIMENT_NAME) upload_tb_logs_to_s3(experiment_name=EXPERIMENT_NAME)