def test_concat_multiindex_sort(self): # SPARK-39314: Respect ps.concat sort parameter to follow pandas behavior idx = pd.MultiIndex.from_tuples([("Y", "A"), ("Y", "B"), ("X", "C"), ("X", "D")]) pdf = pd.DataFrame([[1, 2, 3, 4], [5, 6, 7, 8]], columns=idx) psdf = ps.from_pandas(pdf) ignore_indexes = [True, False] joins = ["inner", "outer"] sorts = [True] if LooseVersion(pd.__version__) >= LooseVersion("1.4"): sorts += [False] objs = [ ([psdf, psdf.reset_index()], [pdf, pdf.reset_index()]), ([psdf.reset_index(), psdf], [pdf.reset_index(), pdf]), ] for ignore_index, join, sort in itertools.product( ignore_indexes, joins, sorts): for i, (psdfs, pdfs) in enumerate(objs): self.assert_eq( ps.concat(psdfs, ignore_index=ignore_index, join=join, sort=sort), pd.concat(pdfs, ignore_index=ignore_index, join=join, sort=sort), )
def complex_psdf(self): pssers = { "this_array": self.psser, "that_array": ps.Series([[2, 3, 4]]), "this_struct": ps.Index([("x", 1)]).to_series().reset_index(drop=True), "that_struct": ps.Index([("a", 2)]).to_series().reset_index(drop=True), } return ps.concat(pssers, axis=1)
def test_concat_column_axis(self): pdf1 = pd.DataFrame({"A": [0, 2, 4], "B": [1, 3, 5]}, index=[1, 2, 3]) pdf1.columns.names = ["AB"] pdf2 = pd.DataFrame({"C": [1, 2, 3], "D": [4, 5, 6]}, index=[1, 3, 5]) pdf2.columns.names = ["CD"] kdf1 = ps.from_pandas(pdf1) kdf2 = ps.from_pandas(pdf2) kdf3 = kdf1.copy() kdf4 = kdf2.copy() pdf3 = pdf1.copy() pdf4 = pdf2.copy() columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B")], names=["X", "AB"]) pdf3.columns = columns kdf3.columns = columns columns = pd.MultiIndex.from_tuples([("X", "C"), ("X", "D")], names=["Y", "CD"]) pdf4.columns = columns kdf4.columns = columns ignore_indexes = [True, False] joins = ["inner", "outer"] objs = [ ([kdf1.A, kdf1.A.rename("B")], [pdf1.A, pdf1.A.rename("B")]), ([kdf3[("X", "A")], kdf3[("X", "B")]], [pdf3[("X", "A")], pdf3[("X", "B")]],), ( [kdf3[("X", "A")], kdf3[("X", "B")].rename("ABC")], [pdf3[("X", "A")], pdf3[("X", "B")].rename("ABC")], ), ( [kdf3[("X", "A")].rename("ABC"), kdf3[("X", "B")]], [pdf3[("X", "A")].rename("ABC"), pdf3[("X", "B")]], ), ] for ignore_index, join in itertools.product(ignore_indexes, joins): for i, (kdfs, pdfs) in enumerate(objs): with self.subTest(ignore_index=ignore_index, join=join, pdfs=pdfs, pair=i): actual = ps.concat(kdfs, axis=1, ignore_index=ignore_index, join=join) expected = pd.concat(pdfs, axis=1, ignore_index=ignore_index, join=join) self.assert_eq( repr(actual.sort_values(list(actual.columns)).reset_index(drop=True)), repr(expected.sort_values(list(expected.columns)).reset_index(drop=True)), )
def test_concat_index_axis(self): pdf = pd.DataFrame({"A": [0, 2, 4], "B": [1, 3, 5], "C": [6, 7, 8]}) # TODO: pdf.columns.names = ["ABC"] psdf = ps.from_pandas(pdf) ignore_indexes = [True, False] joins = ["inner", "outer"] sorts = [True, False] objs = [ ([psdf, psdf], [pdf, pdf]), ([psdf, psdf.reset_index()], [pdf, pdf.reset_index()]), ([psdf.reset_index(), psdf], [pdf.reset_index(), pdf]), ([psdf, psdf[["C", "A"]]], [pdf, pdf[["C", "A"]]]), ([psdf[["C", "A"]], psdf], [pdf[["C", "A"]], pdf]), ([psdf, psdf["C"]], [pdf, pdf["C"]]), ([psdf["C"], psdf], [pdf["C"], pdf]), ([psdf["C"], psdf, psdf["A"]], [pdf["C"], pdf, pdf["A"]]), ([psdf, psdf["C"], psdf["A"]], [pdf, pdf["C"], pdf["A"]]), ] for ignore_index, join, sort in itertools.product( ignore_indexes, joins, sorts): for i, (psdfs, pdfs) in enumerate(objs): with self.subTest(ignore_index=ignore_index, join=join, sort=sort, pdfs=pdfs, pair=i): self.assert_eq( ps.concat(psdfs, ignore_index=ignore_index, join=join, sort=sort), pd.concat(pdfs, ignore_index=ignore_index, join=join, sort=sort), almost=(join == "outer"), ) self.assertRaisesRegex(TypeError, "first argument must be", lambda: ps.concat(psdf)) self.assertRaisesRegex(TypeError, "cannot concatenate object", lambda: ps.concat([psdf, 1])) psdf2 = psdf.set_index("B", append=True) self.assertRaisesRegex(ValueError, "Index type and names should be same", lambda: ps.concat([psdf, psdf2])) self.assertRaisesRegex(ValueError, "No objects to concatenate", lambda: ps.concat([])) self.assertRaisesRegex(ValueError, "All objects passed", lambda: ps.concat([None, None])) pdf3 = pdf.copy() psdf3 = psdf.copy() columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C")]) # TODO: colums.names = ["XYZ", "ABC"] pdf3.columns = columns psdf3.columns = columns objs = [ ([psdf3, psdf3], [pdf3, pdf3]), ([psdf3, psdf3.reset_index()], [pdf3, pdf3.reset_index()]), ([psdf3.reset_index(), psdf3], [pdf3.reset_index(), pdf3]), ([psdf3, psdf3[[("Y", "C"), ("X", "A")]]], [pdf3, pdf3[[("Y", "C"), ("X", "A")]]]), ([psdf3[[("Y", "C"), ("X", "A")]], psdf3], [pdf3[[("Y", "C"), ("X", "A")]], pdf3]), ] for ignore_index, sort in itertools.product(ignore_indexes, sorts): for i, (psdfs, pdfs) in enumerate(objs): with self.subTest(ignore_index=ignore_index, join="outer", sort=sort, pdfs=pdfs, pair=i): self.assert_eq( ps.concat(psdfs, ignore_index=ignore_index, join="outer", sort=sort), pd.concat(pdfs, ignore_index=ignore_index, join="outer", sort=sort), ) # Skip tests for `join="inner" and sort=False` since pandas is flaky. for ignore_index in ignore_indexes: for i, (psdfs, pdfs) in enumerate(objs): with self.subTest(ignore_index=ignore_index, join="inner", sort=True, pdfs=pdfs, pair=i): self.assert_eq( ps.concat(psdfs, ignore_index=ignore_index, join="inner", sort=True), pd.concat(pdfs, ignore_index=ignore_index, join="inner", sort=True), ) self.assertRaisesRegex( ValueError, "MultiIndex columns should have the same levels", lambda: ps.concat([psdf, psdf3]), ) self.assertRaisesRegex( ValueError, "MultiIndex columns should have the same levels", lambda: ps.concat([psdf3[("Y", "C")], psdf3]), ) pdf4 = pd.DataFrame({ "A": [0, 2, 4], "B": [1, 3, 5], "C": [10, 20, 30] }) psdf4 = ps.from_pandas(pdf4) self.assertRaisesRegex( ValueError, r"Only can inner \(intersect\) or outer \(union\) join the other axis.", lambda: ps.concat([psdf, psdf4], join=""), ) self.assertRaisesRegex( ValueError, r"Only can inner \(intersect\) or outer \(union\) join the other axis.", lambda: ps.concat([psdf, psdf4], join="", axis=1), ) self.assertRaisesRegex( ValueError, r"Only can inner \(intersect\) or outer \(union\) join the other axis.", lambda: ps.concat([psdf.A, psdf4.B], join="", axis=1), ) self.assertRaisesRegex( ValueError, r"Labels have to be unique; however, got duplicated labels \['A'\].", lambda: ps.concat([psdf.A, psdf4.A], join="inner", axis=1), )