def evaluate_cv_course(label_type, k=5, label_col = "label_type", raw_data_dir = "morf-data/", course_col = "course", fold_col = "fold_num", pred_cols = ("prob", "pred"), user_col = "userID"): """ Fetch metrics by first averaging over folds within course, then returning results by course. :param label_type: label type defined by user. :param label_col: column containing labels. :param raw_data_bucket: bucket containing raw data; used to fetch course names. :param raw_data_dir: path to directory in raw_data_bucket containing course-level directories. :param proc_data_bucket: bucket containing session-level archived results from [mode] jobs (i.e., session-level extracted features). :param course_col: column containing course identifier. :param pred_cols: user-supplied prediction columns; these columns will be checked for missing values and to ensure they contain values for every user in the course. :param user_col: column containing user ID for predictions. :param labels_file: name of csv file containing labels. :return: None. """ job_config = MorfJobConfig(CONFIG_FILENAME) job_config.update_mode(mode) check_label_type(label_type) raw_data_buckets = job_config.raw_data_buckets proc_data_bucket = job_config.proc_data_bucket s3 = job_config.initialize_s3() # clear any preexisting data for this user/job/mode clear_s3_subdirectory(job_config) course_data = [] for raw_data_bucket in raw_data_buckets: pred_file = generate_archive_filename(job_config, mode="test", extension="csv") pred_key = make_s3_key_path(job_config, pred_file, mode="test") # download course prediction and label files, fetch classification metrics at course level with tempfile.TemporaryDirectory(dir=os.getcwd()) as working_dir: pred_csv = download_from_s3(proc_data_bucket, pred_key, s3, working_dir, job_config=job_config) job_config.update_mode("cv") # set mode to cv to fetch correct labels for sessions even if they are train/test sessions label_csv = initialize_labels(job_config, raw_data_bucket, None, None, label_type, working_dir, raw_data_dir, level="all") pred_df = pd.read_csv(pred_csv) lab_df = pd.read_csv(label_csv, dtype=object) pred_lab_df = pd.merge(lab_df, pred_df, how = "left", on = [user_col, course_col]) check_dataframe_complete(pred_lab_df, job_config, columns = list(pred_cols)) for course in fetch_complete_courses(job_config, data_bucket = raw_data_bucket, data_dir = raw_data_dir, n_train=1): fold_metrics_list = list() for fold_num in range(1, k+1): fold_metrics_df = fetch_binary_classification_metrics(job_config, pred_lab_df[pred_lab_df[fold_col] == fold_num], course) fold_metrics_list.append(fold_metrics_df) assert len(fold_metrics_list) == k, "something is wrong; number of folds doesn't match. Try running job again from scratch." course_metrics_df = pd.concat(fold_metrics_list).mean() course_metrics_df[course_col] = course course_data.append(course_metrics_df) job_config.update_mode(mode) master_metrics_df = pd.concat(course_data, axis = 1).T # reorder dataframe so course name is first cols = list(master_metrics_df) # move the column to head of list using index, pop and insert cols.insert(0, cols.pop(cols.index(course_col))) master_metrics_df = master_metrics_df.ix[:, cols] csv_fp = generate_archive_filename(job_config, extension="csv") master_metrics_df[course_col] = hash_df_column(master_metrics_df[course_col], job_config.user_id, job_config.hash_secret) master_metrics_df.to_csv(csv_fp, index = False, header = True) upload_key = make_s3_key_path(job_config, mode = "test", filename=csv_fp) upload_file_to_s3(csv_fp, bucket=proc_data_bucket, key=upload_key) os.remove(csv_fp) return
def evaluate_course(label_type, label_col = "label_type", raw_data_dir = "morf-data/", course_col = "course", pred_cols = ("prob", "pred"), user_col = "userID", labels_file = "labels-test.csv"): """ Fetch metrics by course. :param label_type: label type defined by user. :param label_col: column containing labels. :param raw_data_bucket: bucket containing raw data; used to fetch course names. :param raw_data_dir: path to directory in raw_data_bucket containing course-level directories. :param proc_data_bucket: bucket containing session-level archived results from [mode] jobs (i.e., session-level extracted features). :param course_col: column containing course identifier. :param pred_cols: user-supplied prediction columns; these columns will be checked for missing values and to ensure they contain values for every user in the course. :param user_col: column containing user ID for predictions. :param labels_file: name of csv file containing labels. :return: None. """ job_config = MorfJobConfig(CONFIG_FILENAME) job_config.update_mode(mode) check_label_type(label_type) raw_data_buckets = job_config.raw_data_buckets proc_data_bucket = job_config.proc_data_bucket s3 = job_config.initialize_s3() # clear any preexisting data for this user/job/mode clear_s3_subdirectory(job_config) course_data = [] for raw_data_bucket in raw_data_buckets: pred_file = generate_archive_filename(job_config, mode="test", extension="csv") pred_key = "{}/{}/{}/{}".format(job_config.user_id, job_config.job_id, "test", pred_file) label_key = raw_data_dir + labels_file # download course prediction and label files, fetch classification metrics at course level with tempfile.TemporaryDirectory(dir=os.getcwd()) as working_dir: download_from_s3(proc_data_bucket, pred_key, s3, working_dir, job_config=job_config) download_from_s3(raw_data_bucket, label_key, s3, working_dir, job_config=job_config) pred_df = pd.read_csv("/".join([working_dir, pred_file])) lab_df = pd.read_csv("/".join([working_dir, labels_file]), dtype=object) lab_df = lab_df[lab_df[label_col] == label_type].copy() pred_lab_df = pd.merge(lab_df, pred_df, how = "left", on = [user_col, course_col]) check_dataframe_complete(pred_lab_df, job_config, columns = pred_cols) for course in fetch_complete_courses(job_config, data_bucket = raw_data_bucket, data_dir = raw_data_dir, n_train=1): course_metrics_df = fetch_binary_classification_metrics(job_config, pred_lab_df, course) course_data.append(course_metrics_df) master_metrics_df = pd.concat(course_data).reset_index().rename(columns={"index": course_col}) csv_fp = generate_archive_filename(job_config, extension="csv") master_metrics_df[course_col] = hash_df_column(master_metrics_df[course_col], job_config.user_id, job_config.hash_secret) master_metrics_df.to_csv(csv_fp, index = False, header = True) upload_key = make_s3_key_path(job_config, mode = "test", filename=csv_fp) upload_file_to_s3(csv_fp, bucket=proc_data_bucket, key=upload_key) os.remove(csv_fp) return
def evaluate_prule_session(): """ Perform statistical testing for prule analysis. :return: None """ raw_data_dir = "morf-data/" job_config = MorfJobConfig(CONFIG_FILENAME) job_config.update_mode(mode) logger = set_logger_handlers(module_logger, job_config) raw_data_buckets = job_config.raw_data_buckets proc_data_bucket = job_config.proc_data_bucket prule_file = job_config.prule_url s3 = job_config.initialize_s3() # clear any preexisting data for this user/job/mode clear_s3_subdirectory(job_config) with tempfile.TemporaryDirectory(dir=os.getcwd()) as working_dir: input_dir, output_dir = initialize_input_output_dirs(working_dir) # pull extraction results from every course into working_dir for raw_data_bucket in raw_data_buckets: for course in fetch_courses(job_config, raw_data_bucket): for session in fetch_sessions(job_config, raw_data_bucket, raw_data_dir, course, fetch_all_sessions=True): if session in fetch_sessions(job_config, raw_data_bucket, raw_data_dir, course): ## session is a non-holdout session fetch_mode = "extract" else: fetch_mode = "extract-holdout" feat_file = generate_archive_filename(job_config, course=course, session=session, mode=fetch_mode) feat_key = make_s3_key_path(job_config, filename=feat_file, course=course, session=session, mode=fetch_mode) feat_local_fp = download_from_s3(proc_data_bucket, feat_key, s3, input_dir, job_config=job_config) unarchive_file(feat_local_fp, input_dir) docker_image_fp = urlparse(job_config.prule_evaluate_image).path docker_image_dir = os.path.dirname(docker_image_fp) docker_image_name = os.path.basename(docker_image_fp) image_uuid = load_docker_image(docker_image_dir, job_config, logger, image_name=docker_image_name) # create a directory for prule file and copy into it; this will be mounted to docker image prule_dir = os.path.join(working_dir, "prule") os.makedirs(prule_dir) shutil.copy(urlparse(prule_file).path, prule_dir) cmd = "{} run --network=\"none\" --rm=true --volume={}:/input --volume={}:/output --volume={}:/prule {} ".format(job_config.docker_exec, input_dir, output_dir, prule_dir, image_uuid) subprocess.call(cmd, shell=True) # rename result file and upload results to s3 final_output_file = os.path.join(output_dir, "output.csv") final_output_archive_name = generate_archive_filename(job_config, extension="csv") final_output_archive_fp = os.path.join(output_dir, final_output_archive_name) os.rename(final_output_file, final_output_archive_fp) output_key = make_s3_key_path(job_config, filename = final_output_archive_name, mode = "test") upload_file_to_s3(final_output_archive_fp, proc_data_bucket, output_key, job_config, remove_on_success=True) return
def cross_validate_session(label_type, k=5, multithread=True, raw_data_dir="morf-data/"): """ Compute k-fold cross-validation across sessions. :return: """ raise NotImplementedError # this is not implemented! # todo: call to create_session_folds() goes here job_config = MorfJobConfig(CONFIG_FILENAME) job_config.update_mode(mode) logger = set_logger_handlers(module_logger, job_config) # clear any preexisting data for this user/job/mode # clear_s3_subdirectory(job_config) if multithread: num_cores = job_config.max_num_cores else: num_cores = 1 logger.info("conducting cross validation") with Pool(num_cores) as pool: for raw_data_bucket in job_config.raw_data_buckets: for course in fetch_complete_courses(job_config, raw_data_bucket): for session in fetch_sessions(job_config, raw_data_bucket, data_dir=raw_data_dir, course=course, fetch_all_sessions=True): for fold_num in range(1, k + 1): with tempfile.TemporaryDirectory( dir=job_config.local_working_directory ) as working_dir: # get fold train data input_dir, output_dir = initialize_input_output_dirs( working_dir) session_input_dir = os.path.join( input_dir, course, session) session_output_dir = os.path.join( output_dir, course, session) trainkey = make_s3_key_path( job_config, course, make_feature_csv_name(course, session, fold_num, "train"), session) train_data_path = download_from_s3( job_config.proc_data_bucket, trainkey, job_config.initialize_s3(), dir=session_input_dir, job_config=job_config) testkey = make_s3_key_path( job_config, course, make_feature_csv_name(course, session, fold_num, "test"), session) test_data_path = download_from_s3( job_config.proc_data_bucket, testkey, job_config.initialize_s3(), dir=session_input_dir, job_config=job_config) # get labels initialize_labels(job_config, raw_data_bucket, course, session, label_type, session_input_dir, raw_data_dir) # run docker image with mode == cv #todo # upload results #todo pool.close() pool.join() return