def test_create_parameter_always_checkpoints(self):
with set_temporary_config({"tasks.defaults.checkpoint": False}):
p = Parameter("p")
assert p.checkpoint is True
def test_getattr_missing(self):
with Flow(name="test") as f:
a = GetAttr()(Parameter("x"), "missing")
state = f.run(parameters=dict(x=DotDict(a=1, b=2, c=3)))
assert isinstance(state.result[a].result, AttributeError)
def test_right_addition(self):
with Flow(name="test") as f:
z = 10 + Parameter("x")
state = f.run(parameters=dict(x=1))
assert state.result[z].result == 11
def test_create_parameter_uses_prefect_result(self):
p = Parameter("p")
assert isinstance(p.result, PrefectResult)
def test_lte_with_constant(self):
with Flow(name="test") as f:
z = Parameter("x") <= 3
state = f.run(parameters=dict(x=2))
assert state.result[z].result is True
def test_getitem_list(self):
with Flow(name="test") as f:
z = Parameter("x")[Parameter("y")]
state = f.run(parameters=dict(x=[1, 2, 3], y=1))
assert state.result[z].result == 2
def get_etl_flow(
username: str = None,
flow_name: str = None,
) -> Flow:
"""
Create an ETL flow to extract data from myfitnesspal into the local database.
Args:
- username (str): MyFitnessPaw username associated with the created workflow
- flow_name (str, optional): An optional name to be applied to the flow
Returns:
- prefect.Flow: The created Prefect flow ready to be run
Raises:
- ValueError: if the `username` keyword argument is not provided
"""
if not username:
raise ValueError("An user must be provided for the flow")
mfp_insertmany = tasks.SQLiteExecuteMany(db=DB_PATH, enforce_fk=True)
flow_name = flow_name or f"MyFitnessPaw ETL <{username.upper()}>"
with Flow(name=flow_name) as etl_flow:
from_date, to_date = tasks.prepare_extraction_start_end_dates(
from_date_str=Parameter(name="from_date", default=None),
to_date_str=Parameter(name="to_date", default=None),
)
measures = Parameter(name="measures", default=["Weight"])
usermail = PrefectSecret(f"MYFITNESSPAL_USERNAME_{username.upper()}")
password = PrefectSecret(f"MYFITNESSPAL_PASSWORD_{username.upper()}")
db_exists = tasks.create_mfp_database()
dates_to_extract = tasks.generate_dates_to_extract(from_date, to_date)
extracted_days = tasks.get_myfitnesspal_day.map(
date=dates_to_extract,
username=unmapped(usermail),
password=unmapped(password),
measures=unmapped(measures),
)
serialized_extracted_days = tasks.serialize_myfitnesspal_days(
extracted_days)
mfp_existing_days = tasks.mfp_select_raw_days(
username=usermail,
dates=dates_to_extract,
upstream_tasks=[db_exists],
)
serialized_days_to_process = tasks.filter_new_or_changed_records(
extracted_records=serialized_extracted_days,
local_records=mfp_existing_days,
)
rawdays_load_state = mfp_insertmany(
query=sql.insert_or_replace_rawdaydata_record,
data=serialized_days_to_process,
)
days_to_process = tasks.deserialize_records_to_process(
serialized_days=serialized_days_to_process,
upstream_tasks=[rawdays_load_state],
)
note_records = tasks.extract_notes(days_to_process)
notes_load_state = mfp_insertmany( # noqa
query=sql.insert_notes,
data=note_records,
)
water_records = tasks.extract_water(days_to_process)
water_load_state = mfp_insertmany( # noqa
query=sql.insert_water,
data=water_records,
)
goal_records = tasks.extract_goals(days_to_process)
goals_load_state = mfp_insertmany( # noqa
query=sql.insert_goals,
data=goal_records,
)
meals_to_process = tasks.extract_meals(days_to_process)
meal_records = tasks.extract_meal_records(meals_to_process)
meals_load_state = mfp_insertmany(
query=sql.insert_meals,
data=meal_records,
)
mealentry_records = tasks.extract_mealentries(meals_to_process)
mealentries_load_state = mfp_insertmany( # noqa
query=sql.insert_mealentries,
data=mealentry_records,
upstream_tasks=[meals_load_state],
)
cardio_records = tasks.extract_cardio_exercises(days_to_process)
cardio_load_state = mfp_insertmany( # noqa
query=sql.insert_cardioexercises,
data=cardio_records,
)
strength_records = tasks.extract_strength_exercises(days_to_process)
strength_load_state = mfp_insertmany( # noqa
query=sql.insert_strengthexercises,
data=strength_records,
)
measurements_records = tasks.extract_measures(days_to_process)
measurements_load_state = mfp_insertmany( # noqa
query=sql.insert_measurements,
data=measurements_records,
)
return etl_flow
def test_right_pow(self):
with Flow(name="test") as f:
z = 10**Parameter("x")
state = f.run(parameters=dict(x=2))
assert state.result[z].result == 100
def test_serialize_parameter():
p = Parameter(name="p")
ps = ParameterSchema().dump(p)
assert ps["default"] == None
assert ps["required"] is True
def test_deserialize_parameter():
p = Parameter(name="p")
p2 = ParameterSchema().load(ParameterSchema().dump(p))
assert isinstance(p2, Parameter)
def test_getattr_dynamic(self):
with Flow(name="test") as f:
z = GetAttr()(Parameter("x"), Parameter("y"))
state = f.run(parameters=dict(x=DotDict(a=1, b=2, c=3), y="b"))
assert state.result[z].result == 2
def test_getattr_nested(self):
with Flow(name="test") as f:
z = GetAttr()(Parameter("x"), "a.b.c")
state = f.run(parameters=dict(x=DotDict(a=DotDict(b=DotDict(c=1)))))
assert state.result[z].result == 1
def test_getattr_constant(self):
with Flow(name="test") as f:
z = GetAttr()(Parameter("x"), "b")
state = f.run(parameters=dict(x=DotDict(a=1, b=2, c=3)))
assert state.result[z].result == 2
def test_pow(self):
with Flow(name="test") as f:
z = Parameter("x")**Parameter("y")
state = f.run(parameters=dict(x=5, y=2))
assert state.result[z].result == 25
def test_multiplication(self):
with Flow(name="test") as f:
z = Parameter("x") * Parameter("y")
state = f.run(parameters=dict(x=2, y=3))
assert state.result[z].result == 6
def test_pow_with_constant(self):
with Flow(name="test") as f:
z = Parameter("x")**3
state = f.run(parameters=dict(x=2))
assert state.result[z].result == 8
def test_multiplication_with_constant(self):
with Flow(name="test") as f:
z = Parameter("x") * 10
state = f.run(parameters=dict(x=2))
assert state.result[z].result == 20
def test_lte(self):
with Flow(name="test") as f:
z = Parameter("x") <= Parameter("y")
state = f.run(parameters=dict(x=5, y=2))
assert state.result[z].result is False
def test_getitem_dict(self):
with Flow(name="test") as f:
z = Parameter("x")[Parameter("y")]
state = f.run(parameters=dict(x=dict(a=1, b=2, c=3), y="b"))
assert state.result[z].result == 2
def test_right_lte(self):
with Flow(name="test") as f:
z = 10 <= Parameter("x")
state = f.run(parameters=dict(x=10))
assert state.result[z].result is True
def test_floor_division_with_constant(self):
with Flow(name="test") as f:
z = Parameter("x") // 10
state = f.run(parameters=dict(x=38))
assert state.result[z].result == 3
def test_addition_with_constant(self):
with Flow(name="test") as f:
z = Parameter("x") + 10
state = f.run(parameters=dict(x=1))
assert state.result[z].result == 11
def test_right_floor_division(self):
with Flow(name="test") as f:
z = 10 // Parameter("x")
state = f.run(parameters=dict(x=4))
assert state.result[z].result == 2
def test_subtraction(self):
with Flow(name="test") as f:
z = Parameter("x") - Parameter("y")
state = f.run(parameters=dict(x=1, y=2))
assert state.result[z].result == -1
def test_mod_with_constant(self):
with Flow(name="test") as f:
z = Parameter("x") % 10
state = f.run(parameters=dict(x=12))
assert state.result[z].result == 2
def test_create_parameter_uses_json_result_handler(self):
p = Parameter("p")
assert isinstance(p.result_handler, JSONResultHandler)
def test_getitem_constant(self):
with Flow(name="test") as f:
z = Parameter("x")["b"]
state = f.run(parameters=dict(x=dict(a=1, b=2, c=3)))
assert state.result[z].result == 2
def test_parameter_serialization(self):
p = Parameter(name="p")
serialized = p.serialize()
assert serialized["name"] == "p"
assert serialized["default"] is None
assert serialized["required"] is True
def test_serialize_parameter_sorts_tags():
assert TaskSchema().dump(Parameter(name="p", tags=["b", "a", "c"]))["tags"] == [
"a",
"b",
"c",
]
