def test_insert_to_bq_options(testing_table_id):
# Override survey data, but make sure to deep copy to prevent mutating state
# in other tests.
# https://apihelp.alchemer.com/help/surveyresponse-per-question-v5#textboxlist
base = copy.deepcopy(EXAMPLE_RESPONSE["data"][0])
base["survey_data"] = {
"37": {
"id": 37,
"type": "parent",
"question": "Textbox List Question Title",
"section_id": 3,
"options": {
"10068": {
"id": 10068,
"option": "Row 1",
"answer": "text list answer"
}
},
"shown": True,
},
"38": {
"id": 38,
"type": "parent",
"question": "Continuous Sum Question Title",
"section_id": 3,
"options": {
"10070": {
"id": 10070,
"option": "Row 1",
"answer": "6"
},
"10071": {
"id": 10071,
"option": "Row 2",
"answer": "7"
},
},
"shown": True,
},
}
transformed = [format_responses(base, SUBMISSION_DATE)]
insert_to_bq(transformed, testing_table_id, SUBMISSION_DATE)
def backfill(start_date, end_date, survey_id, api_token, api_secret,
destination_table):
"""Import data from alchemer (surveygizmo) surveys into BigQuery.
The date range is inclusive of the start and end values.
"""
print(f"Runing backfill of {survey_id} from {start_date} to {end_date}"
" into {destination_table}")
days = (end_date - start_date).days + 1
start = datetime.utcnow()
for i in range(days):
current_date = (start_date + timedelta(i)).isoformat()[:10]
print(f"Running for {current_date}")
survey_data = get_survey_data(survey_id, current_date, api_token,
api_secret)
if not survey_data:
print("No data, skipping insertion...")
continue
insert_to_bq(survey_data, destination_table, current_date)
print(
f"Processed {days} days in {int((datetime.utcnow()-start).total_seconds())} seconds"
)
def test_insert_to_bq_subquestions(testing_table_id):
# Override survey data. Note that the subquestion object is incompatible.
# https://apihelp.alchemer.com/help/surveyresponse-per-question-v5#checkboxgrid
base = copy.deepcopy(EXAMPLE_RESPONSE["data"][0])
base["survey_data"] = {
"30": {
"id": 30,
"type": "parent",
"question": "Checkbox Grid Question Title",
"subquestions": {
"31": {
"10062": {
"id": 10062,
"type": "CHECKBOX",
"parent": 30,
"question": "Row 1 : Column 1",
"answer": "Column 1",
"shown": True,
},
"10063": {
"id": 10063,
"type": "CHECKBOX",
"parent": 30,
"question": "Row 1 : Column 2",
"answer": None,
"shown": True,
},
},
"32": {
"10062": {
"id": 10062,
"type": "CHECKBOX",
"parent": 30,
"question": "Row 2 : Column 1",
"answer": None,
"shown": True,
},
"10063": {
"id": 10063,
"type": "CHECKBOX",
"parent": 30,
"question": "Row 2 : Column 2",
"answer": "Column 2",
"shown": True,
},
},
},
"section_id": 3,
"shown": True,
},
"83": {
"id": 83,
"type": "parent",
"question": "Custom Table Question Title",
"subquestions": {
"10001": {
"id": 10001,
"type": "RADIO",
"question": "Radio Button Column",
"section_id": 4,
"answer": "Option 1",
"answer_id": 10113,
"shown": True,
},
"10002": {
"id": 10002,
"type": "RADIO",
"question": "Radio Button Column",
"section_id": 4,
"answer": "Option 2",
"answer_id": 10114,
"shown": True,
},
},
"section_id": 4,
"shown": True,
},
}
transformed = [format_responses(base, SUBMISSION_DATE)]
insert_to_bq(transformed, testing_table_id, SUBMISSION_DATE)
def test_insert_to_bq(testing_table_id):
transformed = construct_data(EXAMPLE_RESPONSE, SUBMISSION_DATE)
insert_to_bq(transformed, testing_table_id, SUBMISSION_DATE)