def test_tune_simple(): def t1(a: int, b: int) -> float: return a + b for distributable in [True, False, None]: with FugueWorkflow() as dag: df = space_to_df(dag, Space(a=Grid(0, 1), b=Grid(2, 3))) tune(df, t1, distributable=distributable).show() @tunable() def t2(e: ExecutionEngine, a: int, b: int) -> float: assert isinstance(e, ExecutionEngine) return a + b for distributable in [False, None]: with FugueWorkflow() as dag: df = space_to_df(dag, Space(a=Grid(0, 1), b=Grid(2, 3))) tune(df, t2, distributable=distributable).show() # equivalent syntax sugar with FugueWorkflow() as dag: t2.space(a=Grid(0, 1), b=Grid(2, 3)).tune(dag).show() with raises(FugueTuneCompileError): with FugueWorkflow() as dag: df = space_to_df(dag, Space(a=Grid(0, 1), b=Grid(2, 3))) tune(df, t2, distributable=True).show()
def test_space_to_df(): with FugueWorkflow() as dag: df = space_to_df(dag, Space(a=Grid(0, 1), b=Grid(2, 3))) df.assert_eq( dag.df( [ ['[{"a": 0, "b": 2}]'], ['[{"a": 0, "b": 3}]'], ['[{"a": 1, "b": 2}]'], ['[{"a": 1, "b": 3}]'], ], "__fmin_params__:str", )) with FugueWorkflow() as dag: df = space_to_df(dag, Space(a=Grid(0, 1), b=Grid(2, 3)), batch_size=3, shuffle=False) df.assert_eq( dag.df( [ ['[{"a": 0, "b": 2}, {"a": 0, "b": 3}, {"a": 1, "b": 2}]'], ['[{"a": 1, "b": 3}]'], ], "__fmin_params__:str", ))
def test_out_transform(): class OT(OutputTransformer): def process(self, df): return o = _to_output_transformer(OT) w = FugueWorkflow() w.df([[0], [1]], "a:int", data_determiner=to_uuid).out_transform( o, params=dict(n=2) ) assert_eq( """ create [[0],[1]] schema a:int outtransform using OT(n=2) """, w, ) w = FugueWorkflow() w.df([[0], [1]], "a:int", data_determiner=to_uuid).partition( by=["a"], presort="b DESC", num="ROWCOUNT/2" ).out_transform(mock_transformer, params=dict(n=2)) assert_eq( """ create [[0],[1]] schema a:int outtransform prepartition ROWCOUNT / 2 by a presort b desc using mock_transformer(n=2) """, w, )
def test_yield(tmpdir): df = pd.DataFrame([[0, 0]], columns=["a", "b"]) # schema: * def t(df: pd.DataFrame) -> pd.DataFrame: return df.assign(b=df.b + 1) dag = FugueWorkflow() dag.df(df).transform(t).yield_dataframe_as("x") result = dag.run()["x"] assert [[0, 1]] == result.as_array() dag1 = FugueWorkflow() dag1.df(df).transform(t).yield_file_as("x") dag1.run("", {FUGUE_CONF_WORKFLOW_CHECKPOINT_PATH: str(tmpdir)}) dag2 = FugueWorkflow() dag2.df(dag1.yields["x"]).transform(t).yield_dataframe_as("y") result = dag2.run("", {FUGUE_CONF_WORKFLOW_CHECKPOINT_PATH: str(tmpdir)})["y"] assert [[0, 2]] == result.as_array() dag3 = FugueWorkflow() dag3.df(dag2.yields["y"]).transform(t).yield_dataframe_as("z") result = dag3.run()["z"] assert [[0, 3]] == result.as_array()
def test_workflow_determinism_8(): dag1 = FugueWorkflow() a1 = dag1.create_data([[0], [0], [1]], "a:int32") a1.select("a", "b") a1.show() dag2 = FugueWorkflow() a2 = dag2.create_data([[0], [0], [1]], "a:int32") a2.select("a", "b") a2.show() dag3 = FugueWorkflow() a3 = dag3.create_data([[0], [0], [1]], "a:int32") a3.select("b", "a") a3.show() dag4 = FugueWorkflow() a4 = dag4.create_data([[0], [0], [1]], "a:int32") a4.select("a", "b", distinct=True) a4.show() assert a1.spec_uuid() == a2.spec_uuid() assert dag1.spec_uuid() == dag2.spec_uuid() assert a1.spec_uuid() == a3.spec_uuid() assert dag1.spec_uuid() != dag3.spec_uuid() assert a1.spec_uuid() == a4.spec_uuid() assert dag1.spec_uuid() != dag4.spec_uuid()
def test_compile_conf(): def assert_conf(e: ExecutionEngine, **kwargs) -> pd.DataFrame: for k, v in kwargs.items(): assert e.compile_conf[k] == v return pd.DataFrame([[0]], columns=["a"]) dag = FugueWorkflow(conf={"a": 1}) dag.create(assert_conf, params=dict(a=1)) dag.run() with raises(KeyError): # non-compile time param doesn't keep in new engine dag.run(NativeExecutionEngine()) dag = FugueWorkflow(conf={FUGUE_CONF_WORKFLOW_EXCEPTION_HIDE: "abc"}) dag.create(assert_conf, params=dict({FUGUE_CONF_WORKFLOW_EXCEPTION_HIDE: "abc"})) dag.run() # non-compile time param is kepts dag.run(NativeExecutionEngine()) # non-compile time param can't be changed by new engines # new engine compile conf will be overwritten dag.run(NativeExecutionEngine({FUGUE_CONF_WORKFLOW_EXCEPTION_HIDE: "def"}))
def test_zip(): dag = FugueWorkflow() a1 = dag.create(mock_create1, params=dict(n=1)) a2 = dag.create(mock_create1, params=dict(n=2)) a1.zip(a2) assert_eq( """ a=create using mock_create1 params n:1 zip a,(create using mock_create1 params n:2) """, dag, ) dag = FugueWorkflow() a1 = dag.create(mock_create1, params=dict(n=1)) a2 = dag.create(mock_create1, params=dict(n=2)) a1.zip(a2, how="left_outer", partition=dict(by=["a"], presort="b DESC")) assert_eq( """ a=create using mock_create1 params n:1 zip a,(create using mock_create1 params n:2) left outer by a presort b desc """, dag, )
def test_create(): id0 = FugueWorkflow().df([[0]], "a:int32").workflow.spec_uuid() id1 = FugueWorkflow().df([[1]], "a:int32").workflow.spec_uuid() id2 = FugueWorkflow().df([[1]], "a:int32").workflow.spec_uuid() assert id1 != id0 assert id1 == id2
def test_worflow_dataframes(): dag1 = FugueWorkflow() df1 = dag1.df([[0]], "a:int") df2 = dag1.df([[0]], "b:int") dag2 = FugueWorkflow() df3 = dag2.df([[0]], "a:int") dfs1 = WorkflowDataFrames(a=df1, b=df2) assert dfs1["a"] is df1 assert dfs1["b"] is df2 dfs2 = WorkflowDataFrames(dfs1, aa=df1, bb=df2) assert 4 == len(dfs2) with raises(ValueError): WorkflowDataFrames(a=df1, b=df3) with raises(ValueError): WorkflowDataFrames(a=df1, b=ArrayDataFrame([[0]], "a:int")) dag = FugueWorkflow() df = dag.df([[0, 1], [1, 1]], "a:int,b:int") assert df.partition_spec.empty df2 = df.partition(by=["a"]) assert df.partition_spec.empty assert df2.partition_spec == PartitionSpec(by=["a"]) df3 = df.partition_by("a", "b") assert df.partition_spec.empty assert df3.partition_spec == PartitionSpec(by=["a", "b"]) df4 = df.per_partition_by("a", "b") assert df.partition_spec.empty assert df4.partition_spec == PartitionSpec(by=["a", "b"], algo="even") df4 = df.per_row() assert df.partition_spec.empty assert df4.partition_spec == PartitionSpec("per_row")
def test_input_module(): # pylint: disable=no-value-for-parameter @module() def input1(wf: FugueWorkflow) -> WorkflowDataFrame: return wf.df([[0]], "a:int") @module() def input2(wf: FugueWorkflow, a: int) -> WorkflowDataFrame: return wf.df([[a]], "a:int") @module() def input3(wf: FugueWorkflow, a: int, b: int) -> WorkflowDataFrames: return WorkflowDataFrames(a=wf.df([[a]], "a:int"), b=wf.df([[b]], "b:int")) assert not input1.has_input assert input1.has_single_output assert not input1.has_no_output assert input3.has_multiple_output with FugueWorkflow() as dag: input1(dag).assert_eq(dag.df([[0]], "a:int")) with FugueWorkflow() as dag: input2(a=10, wf=dag).assert_eq(dag.df([[10]], "a:int")) with FugueWorkflow() as dag: dfs = input3(dag, 10, 11) dfs["a"].assert_eq(dag.df([[10]], "a:int")) dfs["b"].assert_eq(dag.df([[11]], "b:int"))
def test_visualize_top_n(tmpdir): def t1(a: int, b: int) -> float: return a + b with FugueWorkflow() as dag: df = space_to_df(dag, Space(a=Grid(0, 1), b=Grid(2, 3))) visualize_top_n(tune(df, t1, distributable=False), top=2) @tunable() def t2(df1: pd.DataFrame, df2: pd.DataFrame, a: int, b: int) -> Dict[str, Any]: return { "error": float(a + b + df1["y"].sum() + df2["y"].sum()), "metadata": { "a": a }, } e = NativeExecutionEngine(conf={FUGUE_TUNE_TEMP_PATH: str(tmpdir)}) with FugueWorkflow(e) as dag: df1 = dag.df([[0, 1], [1, 2], [0, 2]], "x:int,y:int").partition(by=["x"]) df2 = dag.df([[0, 10], [1, 20]], "x:int,y:int").partition(by=["x"]) res = t2.space(df1=df1, df2=df2, a=Grid(0, 1), b=Grid(2, 3)).tune() visualize_top_n(res, top=2)
def test_process_module(): # pylint: disable=no-value-for-parameter def process(df: pd.DataFrame, d: int = 1) -> pd.DataFrame: df["a"] += d return df @module def p1(wf: FugueSQLWorkflow, df: WorkflowDataFrame) -> WorkflowDataFrame: return df.process(process) @module() def p2(wf: FugueWorkflow, dfs: WorkflowDataFrames, d: int) -> WorkflowDataFrames: return WorkflowDataFrames( {k: v.process(process, params={"d": d}) for k, v in dfs.items()}) @module(as_method=True, name="p4") def p3(df: WorkflowDataFrame) -> WorkflowDataFrame: return df.process(process) assert p1.has_input assert not p1.has_dfs_input assert p2.has_dfs_input with FugueSQLWorkflow() as dag: df = dag.df([[0]], "a:int") p1(df).assert_eq(dag.df([[1]], "a:int")) p1(dag, df).assert_eq(dag.df([[1]], "a:int")) p1(df=df).assert_eq(dag.df([[1]], "a:int")) p1(df=df, wf=dag).assert_eq(dag.df([[1]], "a:int")) with FugueWorkflow() as dag: dfs = WorkflowDataFrames(aa=dag.df([[0]], "a:int"), bb=dag.df([[10]], "a:int")) r = p2(dag, dfs, 1) r["aa"].assert_eq(dag.df([[1]], "a:int")) r["bb"].assert_eq(dag.df([[11]], "a:int")) r = p2(dfs, 1) r["aa"].assert_eq(dag.df([[1]], "a:int")) r["bb"].assert_eq(dag.df([[11]], "a:int")) r = p2(d=1, dfs=dfs, wf=dag) r["aa"].assert_eq(dag.df([[1]], "a:int")) r["bb"].assert_eq(dag.df([[11]], "a:int")) r = p2(d=1, dfs=dfs) r["aa"].assert_eq(dag.df([[1]], "a:int")) r["bb"].assert_eq(dag.df([[11]], "a:int")) with FugueWorkflow() as dag: df = dag.df([[0]], "a:int") p3(df).assert_eq(dag.df([[1]], "a:int")) p3(df=df).assert_eq(dag.df([[1]], "a:int")) df.p4().assert_eq(dag.df([[1]], "a:int"))
def test_transform(): w = (FugueWorkflow().df([[0], [1]], "a:int", data_determiner=to_uuid).transform( mock_transformer, schema=Schema("a:int"), params=dict(n=2))) assert_eq( """ create [[0],[1]] schema a:int transform using mock_transformer(n=2) schema a:int """, w.workflow, ) w = (FugueWorkflow().df([[0], [1]], "a:int", data_determiner=to_uuid).partition( by=["a"], presort="b DESC", num="ROWCOUNT/2").transform(mock_transformer, schema="*", params=dict(n=2))) assert_eq( """ create [[0],[1]] schema a:int transform prepartition ROWCOUNT / 2 by a presort b desc using mock_transformer(n=2) schema * """, w.workflow, ) def _func(a: int, b: int) -> int: return a + b w = (FugueWorkflow().df([[0], [1]], "a:int", data_determiner=to_uuid).partition( by=["a"], presort="b DESC", num="ROWCOUNT/2").transform(mock_transformer, schema="*", params=dict(n=2), callback=_func)) assert_eq( """ create [[0],[1]] schema a:int transform prepartition ROWCOUNT / 2 by a presort b desc using mock_transformer(n=2) schema * callback _func """, w.workflow, )
def test_checkpoint(): id0 = FugueWorkflow().df([[0]], "a:int32").workflow.spec_uuid() id1 = FugueWorkflow().df([[0]], "a:int32").checkpoint().workflow.spec_uuid() id2 = FugueWorkflow().df([[0]], "a:int32").checkpoint("1").workflow.spec_uuid() id3 = FugueWorkflow().df([[0]], "a:int32").checkpoint(1).workflow.spec_uuid() assert id1 != id0 assert id1 != id2 assert id2 != id0 assert id2 == id3
def test_auto_persist(): dag1 = FugueWorkflow(NativeExecutionEngine()) df1 = dag1.df([[0]], "a:int") df1.show() df1.show() id1 = dag1.spec_uuid() dag2 = FugueWorkflow(NativeExecutionEngine({"fugue.workflow.auto_persist": True})) df1 = dag2.df([[0]], "a:int") df1.show() df1.show() id2 = dag2.spec_uuid() dag3 = FugueWorkflow(NativeExecutionEngine()) df1 = dag3.df([[0]], "a:int").weak_checkpoint(level=None) df1.show() df1.show() id3 = dag3.spec_uuid() assert id1 == id2 assert id2 == id3 dag2 = FugueWorkflow( NativeExecutionEngine( { "fugue.workflow.auto_persist": True, "fugue.workflow.auto_persist_value": "abc", } ) ) df1 = dag2.df([[0]], "a:int") df1.show() df1.show() id2 = dag2.spec_uuid() dag3 = FugueWorkflow(NativeExecutionEngine()) df1 = dag3.df([[0]], "a:int").weak_checkpoint(level="abc") df1.show() df1.show() id3 = dag3.spec_uuid() assert id2 == id3 dag1 = FugueWorkflow(NativeExecutionEngine()) df1 = dag1.df([[0]], "a:int") df1.show() id1 = dag1.spec_uuid() dag2 = FugueWorkflow(NativeExecutionEngine({"fugue.workflow.auto_persist": True})) df1 = dag2.df([[0]], "a:int") df1.show() # auto persist will not trigger id2 = dag2.spec_uuid() dag3 = FugueWorkflow(NativeExecutionEngine()) df1 = dag3.df([[0]], "a:int").weak_checkpoint(level=None) df1.show() id3 = dag3.spec_uuid() assert id1 == id2 assert id2 == id3 # checkpoint, including auto_persist doesn't change determinism
def test_workflow_determinism_5(): dag1 = FugueWorkflow() a1 = dag1.create_data([[0], [0], [1]], "a:int32") b1 = a1.transform(mock_tf1, "*,b:int", pre_partition=dict(by=["a"], num=2)) a1.show() dag2 = FugueWorkflow() a2 = dag2.create_data([[0], [0], [1]], "a:int32") b2 = a2.transform(mock_tf1, "*,b:int", pre_partition=dict(by=["a"], num=2)) a2.show(rows=22) # <--- assert a1.spec_uuid() == a2.spec_uuid() assert b1.spec_uuid() == b2.spec_uuid() assert dag1.spec_uuid() != dag2.spec_uuid()
def test_auto_persist(): dag1 = FugueWorkflow(NativeExecutionEngine()) df1 = dag1.df([[0]], "a:int") df1.show() df1.show() id1 = dag1.spec_uuid() dag2 = FugueWorkflow( NativeExecutionEngine({"fugue.workflow.auto_persist": True})) df1 = dag2.df([[0]], "a:int") df1.show() df1.show() id2 = dag2.spec_uuid() dag3 = FugueWorkflow(NativeExecutionEngine()) df1 = dag3.df([[0]], "a:int").persist() df1.show() df1.show() id3 = dag3.spec_uuid() assert id1 != id2 assert id2 == id3 dag2 = FugueWorkflow( NativeExecutionEngine({ "fugue.workflow.auto_persist": True, "fugue.workflow.auto_persist_value": "abc" })) df1 = dag2.df([[0]], "a:int") df1.show() df1.show() id2 = dag2.spec_uuid() dag3 = FugueWorkflow(NativeExecutionEngine()) df1 = dag3.df([[0]], "a:int").persist("abc") df1.show() df1.show() id3 = dag3.spec_uuid() assert id2 == id3 dag1 = FugueWorkflow(NativeExecutionEngine()) df1 = dag1.df([[0]], "a:int") df1.show() id1 = dag1.spec_uuid() dag2 = FugueWorkflow( NativeExecutionEngine({"fugue.workflow.auto_persist": True})) df1 = dag2.df([[0]], "a:int") df1.show() # auto persist will not trigger id2 = dag2.spec_uuid() dag3 = FugueWorkflow(NativeExecutionEngine()) df1 = dag3.df([[0]], "a:int").persist() df1.show() id3 = dag3.spec_uuid() assert id1 == id2 assert id2 != id3
def test_wf(): @tunable() def func(a: float, b: float, c: int, d: int) -> float: return a * a + b * b + c + d with FugueWorkflow() as dag: space = space_to_df( dag, Space(a=Grid(1, 2), b=Rand(-100, 100), c=Choice(1, -1), d=RandInt(0, 3)), ) tune(space, func, objective_runner=HyperoptRunner(100, seed=3)).show() with FugueWorkflow() as dag: space = space_to_df(dag, Space(a=Grid(1, 2), b=Grid(0, 1), c=1, d=2)) tune(space, func, objective_runner=HyperoptRunner(100, seed=3)).show()
def test_workflow(): builder = FugueWorkflow() a = builder.create_data([[0], [0], [1]], "a:int") raises(InvalidOperationError, lambda: a._task.copy()) raises(InvalidOperationError, lambda: copy.copy(a._task)) raises(InvalidOperationError, lambda: copy.deepcopy(a._task)) a.show() a.show() raises(FugueWorkflowCompileError, lambda: builder.df(123)) b = a.transform(mock_tf1, "*,b:int", pre_partition=dict(by=["a"])) b.show() builder.create_data([[0], [1]], "b:int").show() c = ArrayDataFrame([[100]], "a:int") builder.show(a, b, c) b = a.partition(by=["a"]).transform(mock_tf2).persist().broadcast() b.show() builder.run() df_eq(a.result, [[0], [0], [1]], "a:int") raises(TypeError, lambda: builder.run("abc")) builder.run(FugueWorkflowContext()) df_eq(a.result, [[0], [0], [1]], "a:int") builder.run("NativeExecutionEngine") df_eq(b.result, [[0, 2], [0, 2], [1, 1]], "a:int,b:int") df_eq(b.compute(), [[0, 2], [0, 2], [1, 1]], "a:int,b:int") df_eq(b.compute(NativeExecutionEngine), [[0, 2], [0, 2], [1, 1]], "a:int,b:int")
def test_run_asha(tmpdir): class M(Monitor): def on_report(self, report: TrialReport) -> None: print(report.jsondict) def assert_metric(df: Iterable[Dict[str, Any]], metric: float, ct: int) -> None: n = 0 for row in df: assert row[TUNE_REPORT_METRIC] == metric n += 1 assert n == ct space = Space(a=Grid(0, 1, 2, 3)) dag = FugueWorkflow() dataset = TuneDatasetBuilder(space, str(tmpdir)).build(dag, shuffle=False) obj = F() res = optimize_by_continuous_asha( obj, dataset, plan=[[1.0, 3], [1.0, 2], [1.0, 1], [1.0, 1]], checkpoint_path=str(tmpdir), ) res.result(1).output(assert_metric, dict(metric=1.0, ct=1)) res = optimize_by_continuous_asha( obj, dataset, plan=[[2.0, 2], [1.0, 1], [1.0, 1]], checkpoint_path=str(tmpdir), monitor=M(), ) res.result(1).output(assert_metric, dict(metric=1.0, ct=1)) dag.run()
def test_runtime_exception(): if sys.version_info < (3, 7): return def tr(df: pd.DataFrame) -> pd.DataFrame: raise Exception def show(df): df.show() dag = FugueWorkflow() df = dag.df([[0]], "a:int") df = df.transform(tr, schema="*") show(df) try: dag.run() except: assert len(traceback.extract_tb(sys.exc_info()[2])) < 10 try: dag.run("native", {FUGUE_CONF_WORKFLOW_EXCEPTION_OPTIMIZE: False}) except: assert len(traceback.extract_tb(sys.exc_info()[2])) > 10 try: dag.run("native", {FUGUE_CONF_WORKFLOW_EXCEPTION_HIDE: ""}) except: assert len(traceback.extract_tb(sys.exc_info()[2])) > 10
def test_save_and_use(): dag = FugueWorkflow() a = dag.create(mock_create1, params=dict(n=1)) b = dag.create(mock_create1, params=dict(n=1)) a = a.save_and_use("xx", fmt="parquet", mode="overwrite") b.save_and_use("xx", mode="append") b.save_and_use("xx", mode="error") a = a.save_and_use("xx.csv", fmt="csv", mode="error", single=True, header=True) a = a.partition(by=["x"]).save_and_use("xx", mode="overwrite") dag.create(mock_create1, params=dict(n=2)).save_and_use("xx", mode="overwrite") assert_eq( """ a=create using mock_create1(n=1) b=create using mock_create1(n=1) a=save and use a overwrite parquet "xx" save and use b append "xx" save and use b to "xx" save and use a to single csv "xx.csv"(header=True) save and use prepartition by x overwrite "xx" save and use (create using mock_create1(n=2)) overwrite "xx" """, dag, )
def _process_stack_space(engine: ExecutionEngine, df: DataFrame, keys: List[str], space: Space) -> DataFrame: fe_schema = df.schema.extract(keys) + "__fmin_fe__:str" def _merge_space(df: List[Dict[str, Any]]) -> Iterable[Dict[str, Any]]: p = json.dumps([json.loads(row["__fmin_params__"]) for row in df]) res = df[0] res["__fmin_fe__"] = p yield res # schema: *-__fmin_fe__ def _construct_final_space( df: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]: for row in df: op = json.loads(row["__fmin_params__"]) for o in op: o["_sk__estimators"] = row["__fmin_fe__"] row["__fmin_params__"] = json.dumps(op) yield row with FugueWorkflow(engine) as dag: ddf = dag.df(df) space_df = space_to_df(dag, space).broadcast() if len(keys) == 0: fe = ddf.process(_merge_space, schema=fe_schema) else: fe = ddf.partition(by=keys).transform(_merge_space, schema=fe_schema) result = fe.cross_join(space_df).transform(_construct_final_space) return result.result
def test_hyperband(tmpdir): def assert_metric(df: Iterable[Dict[str, Any]], metric: float, ct: int) -> None: n = 0 for row in df: if metric > 0: assert row[TUNE_REPORT_METRIC] == metric n += 1 assert n == ct space = Space(a=Grid(0, 1, 2, 3)) dag = FugueWorkflow() dataset = TuneDatasetBuilder(space, str(tmpdir)).build(dag) obj = F() res = optimize_by_hyperband( obj, dataset, plans=[ [[1.0, 3], [1.0, 2], [1.0, 1], [1.0, 1]], [[2.0, 2], [1.0, 1], [1.0, 1]], ], checkpoint_path=str(tmpdir), ) res.result().output(assert_metric, dict(metric=0.0, ct=2)) res.result(1).output(assert_metric, dict(metric=1.0, ct=1)) dag.run()
def test_workflow_determinism_6(): dag1 = FugueWorkflow() dag1.create_data([[0], [0], [1]], "a:int32") a1 = dag1.create_data([[0], [0], [1]], "a:int32") b1 = dag1.create_data([[1], [0], [1]], "a:int32") c1 = a1.union(b1) dag2 = FugueWorkflow() a2 = dag1.create_data([[0], [0], [1]], "a:int32") b2 = dag1.create_data([[1], [0], [1]], "a:int32") c2 = a1.union(b2) assert a1.spec_uuid() == a2.spec_uuid() assert b1.spec_uuid() == b2.spec_uuid() assert c1.spec_uuid() == c2.spec_uuid() assert dag1.spec_uuid() != dag2.spec_uuid()
def test_process_stack_space(tmpdir): space1 = ss(LinearRegression, normalize=Grid(True, False)) space2 = ss(LinearRegression, fit_intercept=Grid(True, False)) dag = FugueWorkflow() result0 = build_sk_cv( space1, dag.df(_create_mock_data()), scoring="neg_mean_absolute_error", cv=2, label_col="l", feature_prefix="f_", ).tune(distributable=False, serialize_path=str(tmpdir)) res0 = result0.process(_process_stack_space, params=dict(keys=[], space=space2)) res0.show() result1 = build_sk_cv( space1, dag.df(_create_mock_data()).partition(by=["p"]), scoring="neg_mean_absolute_error", cv=2, label_col="l", feature_prefix="f_", ).tune(distributable=False, serialize_path=str(tmpdir)) res1 = result1.process(_process_stack_space, params=dict(keys=["p"], space=space2)) dag.run() assert 2 == len(res0.result.as_array()) assert 8 == len(res1.result.as_array())
def outputter(df: LocalDataFrame) -> None: keys = [ k for k in df.schema.names if not k.startswith("__df_") and not k.startswith("__fmin_") ] def show(subdf: pd.DataFrame) -> None: if subdf.shape[0] == 0: # pragma: no cover return subdf = subdf.sort_values("__fmin_value__").head(top) title = (json.dumps({k: str(subdf[k].iloc[0]) for k in keys}) if len(keys) > 0 else "") pdf = pd.DataFrame( [json.loads(x) for x in subdf["__fmin_params__"]]) fig = plt.figure(figsize=(12, 3 * len(pdf.columns))) if len(keys) > 0: fig.suptitle( title, va="center", size=15, weight="bold", y=0.93, ) for i in range(len(pdf.columns)): ax = fig.add_subplot(len(pdf.columns), 1, i + 1) pdf[pdf.columns[i]].hist(ax=ax).set_title(pdf.columns[i]) plt.subplots_adjust(hspace=0.5) if len(keys) == 0: show(df.as_pandas()) else: with FugueWorkflow() as dag: dag.df(df).partition(by=keys).out_transform(show)
def test_output_module(): # pylint: disable=no-value-for-parameter @module() def o1(wf: FugueWorkflow, df: WorkflowDataFrame) -> None: pass @module() def o2(wf: FugueWorkflow, df: WorkflowDataFrame): pass @module() def o3(df: WorkflowDataFrame): pass assert o1.has_input assert o1.has_no_output assert o2.has_no_output assert o3.has_no_output with FugueWorkflow() as dag: df = dag.df([[0]], "a:int") o1(df) o1(dag, df) o2(df=df) o2(df=df, wf=dag) o3(df)
def test_workflow_determinism_3(): dag1 = FugueWorkflow() data = [[0], [0], [1]] a1 = dag1.create_data(data, "a:int32", data_determiner=to_uuid) b1 = a1.transform(mock_tf1, "*,b:int", pre_partition=dict(by=["a"], num=2)) a1.show() dag2 = FugueWorkflow() data = [[1], [10], [20]] a2 = dag2.create_data(data, "a:int32", data_determiner=to_uuid) # <--- b2 = a2.transform(mock_tf1, "*,b:int", pre_partition=dict(by=["a"], num=2)) a2.show() assert a1.spec_uuid() != a2.spec_uuid() assert b1.spec_uuid() != b2.spec_uuid() assert dag1.spec_uuid() != dag2.spec_uuid()
def test_create_data(): w = FugueWorkflow().df([[0], [1]], "a:int", data_determiner=to_uuid) assert_eq( """ a=create [[0],[1]] schema a:int """, w.workflow, )