def testLoadShardedCsvFailsOnNonUniformColumns(self): input_csv_dir = tempfile.mkdtemp() _write_to_file("""col1,col2,col3\nA,B,C""", input_csv_dir) _write_to_file("""XXX,YYY,ZZZ\nD,E,F""", input_csv_dir) with self.assertRaisesRegex(ValueError, "YYY"): model_performance_analysis.load_sharded_df_csvs( input_csv_dir, use_given_header=True)
def testLoadShardedCsvFailsOnConflictingArgumentsGiven(self): input_columns = ["letter_1", "letter_2", "letter_3"] with self.assertRaisesRegex(ValueError, "Cannot pass both"): model_performance_analysis.load_sharded_df_csvs( "", column_names=input_columns, use_given_header=True) with self.assertRaisesRegex(ValueError, "Cannot pass both"): model_performance_analysis.load_sharded_df_csvs( "", ignore_first_line=True, use_given_header=True)
def testLoadShardedCsvUseGivenHeader(self): input_csv_dir = tempfile.mkdtemp() _write_to_file("""col1,col2,col3\nA,B,C""", input_csv_dir) _write_to_file("""col1,col2,col3\nD,E,F""", input_csv_dir) expected = pd.read_csv( io.StringIO("col1,col2,col3\n" "A,B,C\n" "D,E,F")) actual = model_performance_analysis.load_sharded_df_csvs( input_csv_dir, use_given_header=True) test_util.assert_dataframes_equal(self, actual, expected, sort_by_column="col1")
def testLoadShardedCsvTest(self): input_csv_dir = tempfile.mkdtemp() _write_to_file("""A,B,C""", input_csv_dir) _write_to_file("""D,E,F""", input_csv_dir) input_columns = ["letter_1", "letter_2", "letter_3"] expected = pd.read_csv( io.StringIO("letter_1,letter_2,letter_3\n" "A,B,C\n" "D,E,F")) actual = model_performance_analysis.load_sharded_df_csvs( input_csv_dir, column_names=input_columns) test_util.assert_dataframes_equal(self, actual, expected, sort_by_column="letter_1")