def make_csv_file():
filenames = []
yield _make_csv_file(filenames)
# Delete csv files that were created
teardown_test_files(filenames)
def make_sql_connection():
"""Sets up sql connections and takes them down after the caller is done.
Yields:
Factory that generates sql connection objects
"""
filenames = []
def _sql_connection(filename, table=""):
# Remove file if exists
if os.path.exists(filename):
os.remove(filename)
filenames.append(filename)
# Create connection and, if needed, table
conn = "sqlite:///{}".format(filename)
if table:
df = pandas.DataFrame({
"col1": [0, 1, 2, 3, 4, 5, 6],
"col2": [7, 8, 9, 10, 11, 12, 13],
"col3": [14, 15, 16, 17, 18, 19, 20],
"col4": [21, 22, 23, 24, 25, 26, 27],
"col5": [0, 0, 0, 0, 0, 0, 0],
})
df.to_sql(table, conn)
return conn
yield _sql_connection
# Teardown the fixture
teardown_test_files(filenames)
def make_csv_file():
"""Pytest fixture factory that makes temp csv files for testing.
Yields:
Function that generates csv files
"""
filenames = []
yield _make_csv_file(filenames)
# Delete csv files that were created
teardown_test_files(filenames)
def TestReadGlobCSVFixture():
filenames = []
base_name = get_unique_filename(extension="")
pytest.glob_path = "{}_*.csv".format(base_name)
pytest.files = ["{}_{}.csv".format(base_name, i) for i in range(11)]
for fname in pytest.files:
# Glob does not guarantee ordering so we have to remove the randomness in the generated csvs.
_make_csv_file(filenames)(fname, row_size=11, remove_randomness=True)
yield
teardown_test_files(filenames)
def test_distributed_pickling(filename, compression):
data = test_data["int_data"]
df = pd.DataFrame(data)
if compression:
filename = f"{filename}.gz"
df.to_pickle_distributed(filename, compression=compression)
pickled_df = pd.read_pickle_distributed(filename, compression=compression)
df_equals(pickled_df, df)
pickle_files = glob.glob(filename)
teardown_test_files(pickle_files)
def test_distributed_pickling(filename, compression):
data = test_data["int_data"]
df = pd.DataFrame(data)
filename_param = filename
if compression:
filename = f"{filename}.gz"
with (warns_that_defaulting_to_pandas() if filename_param
== test_default_to_pickle_filename else nullcontext()):
df.to_pickle_distributed(filename, compression=compression)
pickled_df = pd.read_pickle_distributed(filename,
compression=compression)
df_equals(pickled_df, df)
pickle_files = glob.glob(filename)
teardown_test_files(pickle_files)
def TestReadCSVFixture():
filenames = []
files_ids = [
"test_read_csv_regular",
"test_read_csv_blank_lines",
"test_read_csv_yes_no",
"test_read_csv_nans",
"test_read_csv_bad_lines",
]
# each xdist worker spawned in separate process with separate namespace and dataset
pytest.csvs_names = {
file_id: get_unique_filename()
for file_id in files_ids
}
# test_read_csv_col_handling, test_read_csv_parsing
_make_csv_file(filenames)(
filename=pytest.csvs_names["test_read_csv_regular"], )
# test_read_csv_parsing
_make_csv_file(filenames)(
filename=pytest.csvs_names["test_read_csv_yes_no"],
additional_col_values=["Yes", "true", "No", "false"],
)
# test_read_csv_col_handling
_make_csv_file(filenames)(
filename=pytest.csvs_names["test_read_csv_blank_lines"],
add_blank_lines=True,
)
# test_read_csv_nans_handling
_make_csv_file(filenames)(
filename=pytest.csvs_names["test_read_csv_nans"],
add_blank_lines=True,
additional_col_values=[
"<NA>", "N/A", "NA", "NULL", "custom_nan", "73"
],
)
# test_read_csv_error_handling
_make_csv_file(filenames)(
filename=pytest.csvs_names["test_read_csv_bad_lines"],
add_bad_lines=True,
)
yield
# Delete csv files that were created
teardown_test_files(filenames)
def fixture():
func, filenames = make_default_file(file_type=file_type)
yield func
teardown_test_files(filenames)
