def main():
with Flow("combine-purpleair-sensors") as flow:
environment = Parameter("environment", default="staging")
start = DateTimeParameter("start")
interval_hour = Parameter("interval_hour", default=1)
end = DateTimeParameter("end_inclusive")
dts = datetime_range(start, interval_hour, end)
client = create_purpleair_archive_client(environment)
maybe_all_sensors_processed = extract_warehouse_purpleair_processed.map(
dt=dts, purpleair_client=unmapped(client))
all_sensors_processed = filter_failed(maybe_all_sensors_processed)
combined_sensors = combine_sensors(all_sensors_processed)
blob_client = create_combined_sensors_blob_client(environment)
load_combined_sensors(combined_sensors, blob_client)
# Registers flow to server, which we can then deploy and run in background agents.
# flow.register(project_name="caqi-flows")
# Immediately executes without agents
from datetime import datetime
flow.run(start=datetime(2020, 11, 23, 7),
end_inclusive=datetime(2020, 11, 25, 8))
class TestDateTimeParameter:
@prefect.task
def return_value(x):
return x
with Flow("test") as dt_flow:
dt = DateTimeParameter("dt", required=False)
x = return_value(dt)
def test_datetime_parameter_returns_datetimes_if_passed_datetimes(self):
now = pendulum.now()
state = self.dt_flow.run(dt=now)
assert state.result[self.dt].result == now
assert state.result[self.x].result == now
def test_datetime_parameter_returns_datetimes_if_passed_string(self):
now = pendulum.now()
state = self.dt_flow.run(dt=str(now))
assert state.result[self.dt].result == now
assert state.result[self.x].result == now
def test_datetime_parameter_returns_none_if_passed_none(self):
state = self.dt_flow.run(dt=None)
assert state.result[self.dt].result is None
assert state.result[self.x].result is None
def test_datetime_parameter_with_none_default(self):
state = self.dt_flow.run()
assert state.result[self.dt].result is None
assert state.result[self.x].result is None
def test_datetime_parameter_returns_datetimes_with_default_string(self):
now = pendulum.now()
with Flow("test") as dt_flow:
dt = DateTimeParameter("dt", default=str(now), required=False)
x = self.return_value(dt)
state = dt_flow.run()
assert state.result[dt].result == now
assert state.result[x].result == now
def main():
with Flow("reprocess-purpleair") as flow:
environment = Parameter("environment", default="staging")
start = DateTimeParameter("start")
interval_hour = Parameter("interval_hour", default=1)
end = DateTimeParameter("end_inclusive")
dts = datetime_range(start, interval_hour, end)
client = create_purpleair_archive_client(environment)
all_sensors_raw = extract_warehouse_purpleair.map(
dt=dts, purpleair_client=unmapped(client))
all_sensors_processed = transform_all_sensors_raw.map(all_sensors_raw)
blob_client = create_hour_blob_client.map(
environment=unmapped(environment), dt=dts)
load_all_sensors_processed.map(all_sensors_processed, blob_client)
# Registers flow to server, which we can then deploy and run in background agents.
flow.register(project_name="caqi-flows")
def test_datetime_parameter_must_have_value_or_default_when_required_is_true(
self):
with Flow("test") as dt_flow:
dt = DateTimeParameter("dt", required=True)
x = self.return_value(dt)
with pytest.raises(
ValueError,
match=
"Flow.run did not receive the following required parameters: dt",
):
_ = dt_flow.run()
class TestDateTimeParameter:
@prefect.task
def return_value(x):
return x
with Flow("test") as dt_flow:
dt = DateTimeParameter("dt", required=False)
x = return_value(dt)
def test_datetime_parameter_returns_datetimes_if_passed_datetimes(self):
now = pendulum.now()
state = self.dt_flow.run(dt=now)
assert state.result[self.dt].result == now
assert state.result[self.x].result == now
def test_datetime_parameter_returns_datetimes_if_passed_string(self):
now = pendulum.now()
state = self.dt_flow.run(dt=str(now))
assert state.result[self.dt].result == now
assert state.result[self.x].result == now
def test_datetime_parameter_returns_none_if_passed_none(self):
state = self.dt_flow.run(dt=None)
assert state.result[self.dt].result is None
assert state.result[self.x].result is None
def test_datetime_parameter_with_none_default(self):
state = self.dt_flow.run()
assert state.result[self.dt].result is None
assert state.result[self.x].result is None
def test_datetime_parameter_returns_datetimes_with_default_string(self):
now = pendulum.now()
with Flow("test") as dt_flow:
dt = DateTimeParameter("dt", default=str(now), required=False)
x = self.return_value(dt)
state = dt_flow.run()
assert state.result[dt].result == now
assert state.result[x].result == now
def test_datetime_parameter_must_have_value_or_default_when_required_is_true(
self):
with Flow("test") as dt_flow:
dt = DateTimeParameter("dt", required=True)
x = self.return_value(dt)
with pytest.raises(
ValueError,
match=
"Flow.run did not receive the following required parameters: dt",
):
_ = dt_flow.run()
def main():
with Flow("reprocess-purpleair-single") as flow:
environment = Parameter("environment", default="staging")
dt = DateTimeParameter("dt")
client = create_purpleair_archive_client(environment)
all_sensors_raw = extract_warehouse_purpleair(dt=dt, purpleair_client=client)
all_sensors_processed = transform_all_sensors_raw(all_sensors_raw)
blob_client = create_hour_blob_client(environment=environment, dt=dt)
load_all_sensors_processed(all_sensors_processed, blob_client)
# Registers flow to server, which we can then deploy and run in background agents.
flow.register(project_name="caqi-flows")
def main():
with Flow("mean-aqi") as flow:
environment = Parameter("environment", default="staging")
start = DateTimeParameter("start")
interval_hour = Parameter("interval_hour", default=1)
end = DateTimeParameter("end_inclusive")
dts = datetime_range(start, interval_hour, end)
client = create_purpleair_archive_client(environment)
maybe_all_sensors_processed = extract_warehouse_purpleair_processed.map(dt=dts, purpleair_client=unmapped(client))
all_sensors_processed = filter_failed(maybe_all_sensors_processed)
mean_aqi = transform_processed_mean.map(all_sensors_processed)
combined_mean_aqi = combine_mean_aqis(mean_aqi)
blob_client = create_mean_aqi_blob_client(environment)
load_mean_aqi(combined_mean_aqi, blob_client)
# blob_client = create_hour_blob_client.map(environment=unmapped(environment), dt=dts)
# load_all_sensors_processed.map(all_sensors_processed, blob_client)
# Registers flow to server, which we can then deploy and run in background agents.
flow.register(project_name="caqi-flows")