def test_run_with_no_db(self):
sqlite_script = SQLiteScript(script="SELECT * FROM table")
msg_match = "The db must be specified"
with pytest.raises(ValueError, match=msg_match):
sqlite_script.run()
def test_run_with_no_script(self):
sqlite_script = SQLiteScript(db="database.db")
msg_match = "The query script string must be specified"
with pytest.raises(ValueError, match=msg_match):
sqlite_script.run()
from prefect.engine.result_handlers import LocalResultHandler
from prefect.tasks.database.sqlite import SQLiteScript
from prefect.schedules import IntervalSchedule
# Handle changes in state
def alert_failed(obj, old_state, new_state):
'''Must match this signature with obj, old, new'''
if new_state.is_failed():
print("!!! Failed !!!")
# Setup
create_table = SQLiteScript(db='cfpbcomplaints.db',
script="""
CREATE TABLE IF NOT EXISTS complaint
(timestamp TEXT, state TEXT, product TEXT, company TEXT, complaint_what_happened TEXT)
""")
# Extract
@task(cache_for=datetime.timedelta(days=1),
state_handlers=[alert_failed],
result_handler=LocalResultHandler())
def get_complaint_data():
r = requests.get(
'https://www.consumerfinance.gov/data-research/consumer-complaints/search/api/v1/',
params={'size': 10})
response_json = json.loads(r.text)
print("Actually requested this time!")
return response_json['hits']['hits']