def netflowSolver(modelType): tdf = TicDatFactory(**netflowSchema()) addNetflowForeignKeys(tdf) addNetflowDataTypes(tdf) dat = tdf.copy_tic_dat(netflowData()) assert not tdf.find_data_type_failures( dat) and not tdf.find_foreign_key_failures(dat) mdl = Model(modelType, "netflow") flow = {} for h, i, j in dat.cost: if (i, j) in dat.arcs: flow[h, i, j] = mdl.add_var(name='flow_%s_%s_%s' % (h, i, j)) flowslice = Slicer(flow) for i_, j_ in dat.arcs: mdl.add_constraint(mdl.sum(flow[h, i, j] for h, i, j in flowslice.slice('*', i_, j_)) <= dat.arcs[i_, j_]["capacity"], name='cap_%s_%s' % (i_, j_)) for h_, j_ in set(k for k, v in dat.inflow.items() if abs(v["quantity"]) > 0).union( {(h, i) for h, i, j in flow}, {(h, j) for h, i, j in flow}): mdl.add_constraint( mdl.sum(flow[h, i, j] for h, i, j in flowslice.slice(h_, '*', j_)) + dat.inflow.get((h_, j_), {"quantity": 0})["quantity"] == mdl.sum( flow[h, i, j] for h, i, j in flowslice.slice(h_, j_, '*')), name='node_%s_%s' % (h_, j_)) mdl.set_objective( mdl.sum(flow * dat.cost[h, i, j]["cost"] for (h, i, j), flow in flow.items())) if mdl.optimize(): solutionFactory = TicDatFactory( flow=[["commodity", "source", "destination"], ["quantity"]]) if mdl.optimize(): rtn = solutionFactory.TicDat() for (h, i, j), var in flow.items(): if mdl.get_solution_value(var) > 0: rtn.flow[h, i, j] = mdl.get_solution_value(var) return rtn, sum(dat.cost[h, i, j]["cost"] * r["quantity"] for (h, i, j), r in rtn.flow.items())
def dietSolver(modelType): tdf = TicDatFactory(**dietSchema()) addDietForeignKeys(tdf) addDietDataTypes(tdf) dat = tdf.copy_tic_dat(dietData()) assert not tdf.find_data_type_failures( dat) and not tdf.find_foreign_key_failures(dat) mdl = Model(modelType, "diet") nutrition = {} for c, n in dat.categories.items(): nutrition[c] = mdl.add_var(lb=n["minNutrition"], ub=n["maxNutrition"], name=c) # Create decision variables for the foods to buy buy = {} for f in dat.foods: buy[f] = mdl.add_var(name=f) # Nutrition constraints for c in dat.categories: mdl.add_constraint(mdl.sum(dat.nutritionQuantities[f, c]["qty"] * buy[f] for f in dat.foods) == nutrition[c], name=c) mdl.set_objective(mdl.sum(buy[f] * c["cost"] for f, c in dat.foods.items())) if mdl.optimize(): solutionFactory = TicDatFactory(parameters=[[], ["totalCost"]], buyFood=[["food"], ["qty"]], consumeNutrition=[["category"], ["qty"]]) sln = solutionFactory.TicDat() for f, x in buy.items(): if mdl.get_solution_value(x) > 0.0001: sln.buyFood[f] = mdl.get_solution_value(x) for c, x in nutrition.items(): sln.consumeNutrition[c] = mdl.get_solution_value(x) return sln, sum(dat.foods[f]["cost"] * r["qty"] for f, r in sln.buyFood.items())
def test_true_false(self): if not self.can_run: return tdf = TicDatFactory(table=[["pkf"], ["df1", "df2"]]) tdf.set_data_type("table", "df2", min=-float("inf")) dat = tdf.TicDat(table=[["d1", True, 100], ["d2", False, 200], ["d3", False, -float("inf")]]) self.assertTrue(len(dat.table) == 3) self.assertFalse(tdf.find_data_type_failures(dat)) pgtf = tdf.pgsql ex = None try: pgtf.write_data(None, self.engine, test_schema) except utils.TicDatError as te: ex = str(te) self.assertTrue(ex and "Not a valid TicDat object" in ex) pgtf.write_schema(self.engine, test_schema, forced_field_types={("table", "df1"): "bool"}) pgtf.write_data(dat, self.engine, test_schema) self.assertFalse(pgtf.find_duplicates(self.engine, test_schema)) pg_tic_dat = pgtf.create_tic_dat(self.engine, test_schema) self.assertTrue(tdf._same_data(dat, pg_tic_dat))
def test_ints_and_strings_and_lists(self): if not self.can_run: return tdf = TicDatFactory(t_one=[[], ["str_field", "int_field"]], t_two=[["str_field", "int_field"], []]) for t in tdf.all_tables: tdf.set_data_type(t, "str_field", strings_allowed=['This', 'That'], number_allowed=False) tdf.set_data_type(t, "int_field", must_be_int=True) dat = tdf.TicDat(t_one=[["This", 1], ["That", 2], ["This", 111], ["That", 211]], t_two=[["This", 10], ["That", 9]]) self.assertFalse(tdf.find_data_type_failures(dat)) self.assertTrue(len(dat.t_one) == 4) self.assertTrue(len(dat.t_two) == 2) pgtf = tdf.pgsql pgtf.write_schema(self.engine, test_schema) pgtf.write_data(dat, self.engine, test_schema) self.assertFalse(pgtf.find_duplicates(self.engine, test_schema)) pg_tic_dat = pgtf.create_tic_dat(self.engine, test_schema) self.assertTrue(tdf._same_data(dat, pg_tic_dat))
def _testFantop(modelType, sqlFile): dataFactory = TicDatFactory(parameters=[["Key"], ["Value"]], players=[['Player Name'], [ 'Position', 'Average Draft Position', 'Expected Points', 'Draft Status' ]], roster_requirements=[['Position'], [ 'Min Num Starters', 'Max Num Starters', 'Min Num Reserve', 'Max Num Reserve', 'Flex Status' ]], my_draft_positions=[['Draft Position'], []]) # add foreign key constraints (optional, but helps with preventing garbage-in, garbage-out) dataFactory.add_foreign_key("players", "roster_requirements", ['Position', 'Position']) # set data types (optional, but helps with preventing garbage-in, garbage-out) dataFactory.set_data_type("parameters", "Key", number_allowed=False, strings_allowed=[ "Starter Weight", "Reserve Weight", "Maximum Number of Flex Starters" ]) dataFactory.set_data_type("parameters", "Value", min=0, max=float("inf"), inclusive_min=True, inclusive_max=False) dataFactory.set_data_type("players", "Average Draft Position", min=0, max=float("inf"), inclusive_min=False, inclusive_max=False) dataFactory.set_data_type("players", "Expected Points", min=-float("inf"), max=float("inf"), inclusive_min=False, inclusive_max=False) dataFactory.set_data_type("players", "Draft Status", strings_allowed=[ "Un-drafted", "Drafted By Me", "Drafted By Someone Else" ]) for fld in ("Min Num Starters", "Min Num Reserve", "Max Num Reserve"): dataFactory.set_data_type("roster_requirements", fld, min=0, max=float("inf"), inclusive_min=True, inclusive_max=False, must_be_int=True) dataFactory.set_data_type("roster_requirements", "Max Num Starters", min=0, max=float("inf"), inclusive_min=False, inclusive_max=True, must_be_int=True) dataFactory.set_data_type( "roster_requirements", "Flex Status", number_allowed=False, strings_allowed=["Flex Eligible", "Flex Ineligible"]) dataFactory.set_data_type("my_draft_positions", "Draft Position", min=0, max=float("inf"), inclusive_min=False, inclusive_max=False, must_be_int=True) solutionFactory = TicDatFactory(my_draft=[[ 'Player Name' ], [ 'Draft Position', 'Position', 'Planned Or Actual', 'Starter Or Reserve' ]]) dat = dataFactory.sql.create_tic_dat_from_sql(os.path.join( _codeDir(), sqlFile), freeze_it=True) assert dataFactory.good_tic_dat_object(dat) assert not dataFactory.find_foreign_key_failures(dat) assert not dataFactory.find_data_type_failures(dat) expected_draft_position = {} # for our purposes, its fine to assume all those drafted by someone else are drafted # prior to any players drafted by me for player_name in sorted( dat.players, key=lambda _p: { "Un-drafted": dat.players[_p]["Average Draft Position"], "Drafted By Me": -1, "Drafted By Someone Else": -2 }[dat.players[_p]["Draft Status"]]): expected_draft_position[player_name] = len(expected_draft_position) + 1 assert max(expected_draft_position.values()) == len( set(expected_draft_position.values())) == len(dat.players) assert min(expected_draft_position.values()) == 1 already_drafted_by_me = { player_name for player_name, row in dat.players.items() if row["Draft Status"] == "Drafted By Me" } can_be_drafted_by_me = { player_name for player_name, row in dat.players.items() if row["Draft Status"] != "Drafted By Someone Else" } m = Model(modelType, 'fantop') my_starters = { player_name: m.add_var(type="binary", name="starter_%s" % player_name) for player_name in can_be_drafted_by_me } my_reserves = { player_name: m.add_var(type="binary", name="reserve_%s" % player_name) for player_name in can_be_drafted_by_me } for player_name in can_be_drafted_by_me: if player_name in already_drafted_by_me: m.add_constraint(my_starters[player_name] + my_reserves[player_name] == 1, name="already_drafted_%s" % player_name) else: m.add_constraint( my_starters[player_name] + my_reserves[player_name] <= 1, name="cant_draft_twice_%s" % player_name) for i, draft_position in enumerate(sorted(dat.my_draft_positions)): m.add_constraint(m.sum( my_starters[player_name] + my_reserves[player_name] for player_name in can_be_drafted_by_me if expected_draft_position[player_name] < draft_position) <= i, name="at_most_%s_can_be_ahead_of_%s" % (i, draft_position)) my_draft_size = m.sum(my_starters[player_name] + my_reserves[player_name] for player_name in can_be_drafted_by_me) m.add_constraint(my_draft_size >= len(already_drafted_by_me) + 1, name="need_to_extend_by_at_least_one") m.add_constraint(my_draft_size <= len(dat.my_draft_positions), name="cant_exceed_draft_total") for position, row in dat.roster_requirements.items(): players = { player_name for player_name in can_be_drafted_by_me if dat.players[player_name]["Position"] == position } starters = m.sum(my_starters[player_name] for player_name in players) reserves = m.sum(my_reserves[player_name] for player_name in players) m.add_constraint(starters >= row["Min Num Starters"], name="min_starters_%s" % position) m.add_constraint(starters <= row["Max Num Starters"], name="max_starters_%s" % position) m.add_constraint(reserves >= row["Min Num Reserve"], name="min_reserve_%s" % position) m.add_constraint(reserves <= row["Max Num Reserve"], name="max_reserve_%s" % position) if "Maximum Number of Flex Starters" in dat.parameters: players = { player_name for player_name in can_be_drafted_by_me if dat.roster_requirements[dat.players[player_name]["Position"]] ["Flex Status"] == "Flex Eligible" } m.add_constraint( m.sum(my_starters[player_name] for player_name in players) <= dat.parameters["Maximum Number of Flex Starters"]["Value"], name="max_flex") starter_weight = dat.parameters["Starter Weight"][ "Value"] if "Starter Weight" in dat.parameters else 1 reserve_weight = dat.parameters["Reserve Weight"][ "Value"] if "Reserve Weight" in dat.parameters else 1 m.set_objective(m.sum(dat.players[player_name]["Expected Points"] * (my_starters[player_name] * starter_weight + my_reserves[player_name] * reserve_weight) for player_name in can_be_drafted_by_me), sense="maximize") if not m.optimize(): return sln = solutionFactory.TicDat() def almostone(x): return abs(m.get_solution_value(x) - 1) < 0.0001 picked = sorted([ player_name for player_name in can_be_drafted_by_me if almostone(my_starters[player_name]) or almostone(my_reserves[player_name]) ], key=lambda _p: expected_draft_position[_p]) assert len(picked) <= len(dat.my_draft_positions) if len(picked) < len(dat.my_draft_positions): print( "Your model is over-constrained, and thus only a partial draft was possible" ) draft_yield = 0 for player_name, draft_position in zip(picked, sorted(dat.my_draft_positions)): draft_yield += dat.players[player_name]["Expected Points"] * \ (starter_weight if almostone(my_starters[player_name]) else reserve_weight) assert draft_position <= expected_draft_position[player_name] sln.my_draft[player_name]["Draft Position"] = draft_position sln.my_draft[player_name]["Position"] = dat.players[player_name][ "Position"] sln.my_draft[player_name][ "Planned Or Actual"] = "Actual" if player_name in already_drafted_by_me else "Planned" sln.my_draft[player_name]["Starter Or Reserve"] = \ "Starter" if almostone(my_starters[player_name]) else "Reserve" return sln, draft_yield
def testDateTime(self): schema = test_schema + "_datetime" tdf = TicDatFactory(table_with_stuffs=[["field one"], ["field two"]], parameters=[["a"], ["b"]]) tdf.add_parameter("p1", "Dec 15 1970", datetime=True) tdf.add_parameter("p2", None, datetime=True, nullable=True) tdf.set_data_type("table_with_stuffs", "field one", datetime=True) tdf.set_data_type("table_with_stuffs", "field two", datetime=True, nullable=True) dat = tdf.TicDat(table_with_stuffs=[[ dateutil.parser.parse("July 11 1972"), None ], [datetime.datetime.now(), dateutil.parser.parse("Sept 11 2011")]], parameters=[["p1", "7/11/1911"], ["p2", None]]) self.assertFalse( tdf.find_data_type_failures(dat) or tdf.find_data_row_failures(dat)) tdf.pgsql.write_schema(self.engine, schema) tdf.pgsql.write_data(dat, self.engine, schema) dat_1 = tdf.pgsql.create_tic_dat(self.engine, schema) self.assertFalse( tdf._same_data(dat, dat_1, nans_are_same_for_data_rows=True)) self.assertTrue( all( len(getattr(dat, t)) == len(getattr(dat_1, t)) for t in tdf.all_tables)) self.assertFalse( tdf.find_data_type_failures(dat_1) or tdf.find_data_row_failures(dat_1)) self.assertTrue( isinstance(dat_1.parameters["p1"]["b"], datetime.datetime)) self.assertTrue( all( isinstance(_, datetime.datetime) for _ in dat_1.table_with_stuffs)) self.assertTrue( len([_ for _ in dat_1.table_with_stuffs if pd.isnull(_)]) == 0) self.assertTrue( all( isinstance(_, datetime.datetime) or pd.isnull(_) for v in dat_1.table_with_stuffs.values() for _ in v.values())) self.assertTrue( len([ _ for v in dat_1.table_with_stuffs.values() for _ in v.values() if pd.isnull(_) ]) == 1) pdf = PanDatFactory.create_from_full_schema( tdf.schema(include_ancillary_info=True)) pan_dat = pdf.pgsql.create_pan_dat(self.engine, schema) dat_2 = pdf.copy_to_tic_dat(pan_dat) # pandas can be a real PIA sometimes, hacking around some weird downcasting for k in list(dat_2.table_with_stuffs): dat_2.table_with_stuffs[pd.Timestamp( k)] = dat_2.table_with_stuffs.pop(k) self.assertTrue( tdf._same_data(dat_1, dat_2, nans_are_same_for_data_rows=True)) pdf.pgsql.write_data(pan_dat, self.engine, schema) dat_3 = pdf.copy_to_tic_dat( pdf.pgsql.create_pan_dat(self.engine, schema)) for k in list(dat_3.table_with_stuffs): dat_3.table_with_stuffs[pd.Timestamp( k)] = dat_3.table_with_stuffs.pop(k) self.assertTrue( tdf._same_data(dat_1, dat_3, nans_are_same_for_data_rows=True))