def fill_pdf_template_lambda(
template: PDFTemplate,
data: Dict[str, Any],
item: StorageItem,
file_name: str,
signature: Optional[SecureUploadItem] = None,
):
# Call out to lambda
signature_url = None
if signature:
signature_url = signature.file.url
cleaned_data = clean_data(data)
template_data = serialize_template(template)
with tracer.trace("common.pdf.pdftemplate.save_blank"):
# Save an empty blob to S3 to set the headers, etc., then generate
# a presigned PUT
item.file.save(file_name, ContentFile(""), False)
output_url = s3_client.generate_presigned_url(
"put_object",
Params={
"Bucket": settings.AWS_STORAGE_PRIVATE_BUCKET_NAME, # type:ignore
"Key": os.path.join(item.file.storage.location, item.file.name),
},
)
with tracer.trace("common.pdf.pdftemplate.generate_payload"):
lambda_payload = bytes(
json.dumps({
"template": template_data,
"data": cleaned_data,
"signature": signature_url,
"output": output_url,
}),
"utf-8",
)
with tracer.trace("common.pdf.pdftemplate.call_lambda"):
response = lambda_client.invoke(
FunctionName=settings.PDF_GENERATION_LAMBDA_FUNCTION,
InvocationType="RequestResponse",
LogType="Tail",
Payload=lambda_payload,
)
if response.get("FunctionError"):
logs = base64.b64decode(response["LogResult"]).decode()
msg = f"Error from PDF Filler Lambda function: {response['FunctionError']}"
logger.error(
msg,
extra={"lambda_logs": logs},
)
raise PDFFillerLambdaError(logs, msg)
item.save()
def pull(days: int = None, hours: int = None) -> None:
url = settings.MOVER_LEADS_ENDPOINT
if not days and not hours:
# include an extra hour|day so we don't miss any records
hours = settings.MOVER_PULL_INTERVAL_HOURS + 1
if hours:
url += f"?hrs={hours}"
elif days:
url += f"?days={days}"
url = url.format(client_id=settings.MOVER_ID)
with tracer.trace("mover.leads", service="mover"):
response = requests.get(
url,
headers={
"Authorization": settings.MOVER_SECRET,
"Content-Type": "application/json",
},
)
if response.status_code != 200:
logger.error(
f"Mover endpoint {url} returned {response.status_code}: {response.text}"
)
sentry_sdk.capture_exception(
MoverError(f"Mover endpoint {url} return {response.status_code}"))
return
leads = response.json()["leads"]
logger.info(f"Fetched {len(leads)} leads")
store_leads(leads)
def query_targetsmart(serializer_data):
# TS is pretty conservative, so we'll only pass in the bare minimum needed for a
# match.
address2 = serializer_data.get("address2", "")
address_line = f'{serializer_data["address1"]} {address2}'.strip()
full_address = (
f'{address_line}, {serializer_data["city"]}, {serializer_data["state"]} '
f'{serializer_data["zipcode"]}')
query = {
"first_name": remove_special_characters(serializer_data["first_name"]),
"last_name": remove_special_characters(serializer_data["last_name"]),
"state": serializer_data["state"],
"zip_code": serializer_data["zipcode"],
"unparsed_full_address": remove_special_characters(full_address),
# only use year to match date_of_birth as some states have 1/1 for unknown dates
"dob": serializer_data["date_of_birth"].strftime("%Y*"),
}
session = get_session()
with tracer.trace("ts.registration_check", service="targetsmartapi"):
response = session.get(
TARGETSMART_ENDPOINT,
params=query,
)
if response.status_code != 200:
return {"error": f"HTTP error {response.status_code}: {response.text}"}
return response.json()
def create_form(session: requests.Session):
form_name = f"{settings.ENV}_{settings.MOVER_SOURCE}_lead"
# create form
logger.info(f"Creating form {form_name}")
title = f"VoteAmerica {settings.MOVER_SOURCE} lead"
if settings.ENV != "prod":
title += f" ({settings.ENV})"
with tracer.trace("an.form_create", service="actionnetwork"):
response = session.post(
ACTIONNETWORK_FORM_ENDPOINT,
json={
"identifiers": [f"voteamerica:{form_name}"],
"title": title,
"origin_system": "voteamerica",
},
)
if response.status_code != 200:
logger.error(
f"Failed to create ActionNetwork {form_name} form: {response.status_code} {response.text}"
)
raise ActionNetworkError(
f"Failed to create {form_name} form, status_code {response.status_code}"
)
form_id = get_form_id(response.json()["identifiers"], form_name)
logger.info(f"Created {form_name} form {form_id}")
return form_id
def create_form(session, form_description: str, form_name: str) -> str:
if settings.ENV != "prod":
form_name = f"staging_{form_name}"
form_description += " (staging)"
# create form
logger.info(f"Creating form {form_name}")
with tracer.trace("an.form_create", service="actionnetwork"):
response = session.post(
FORM_ENDPOINT,
json={
"identifiers": [f"voteamerica:{form_name}"],
"title": form_description,
"origin_system": "voteamerica",
},
)
if response.status_code != 200:
logger.error(
f"Failed to create ActionNetwork {form_name} form: {response.status_code} {response.text}"
)
raise ActionNetworkError(
f"Failed to create {form_name} form, status_code {response.status_code}"
)
form_id = get_form_id(response.json()["identifiers"], form_name)
logger.info(f"Created {form_name} form {form_id}")
return form_id
def query_alloy(serializer_data):
address2 = serializer_data.get("address2", "")
if address2 is None:
address2 = ""
address_line = f"{serializer_data['address1']} {address2}".strip()
query = {
"first_name": serializer_data["first_name"],
"last_name": serializer_data["last_name"],
"address": address_line,
"city": serializer_data["city"],
"state": serializer_data["state"],
"zip": serializer_data["zipcode"],
# need full DOB here YYYY-MM-DD
"birth_date": serializer_data["date_of_birth"].strftime("%Y-%m-%d"),
}
session = get_session()
with tracer.trace("alloy.verify", service="alloyapi"):
response = session.get(
ALLOY_ENDPOINT,
params=query,
)
if response.status_code != 200:
return {"error": f"HTTP error {response.status_code}: {response.text}"}
return response.json()
def post_person(info, form_id, api_key, slug):
from common.apm import tracer
session = get_session(api_key)
url = ADD_ENDPOINT.format(form_id=form_id)
try:
with tracer.trace("an.form.submission", service="actionnetwork"):
response = session.post(
url,
json=info,
)
if response.status_code != 200:
extra = {
"url": url,
"info": info,
"status_code": response.status_code
}
logger.info(response.text)
logger.error(
"actionnetwork: Error posting to %(url)s, info %(info)s, status_code %(status_code)s",
extra,
extra=extra,
)
sentry_sdk.capture_exception(
ActionNetworkError(
f"Error posting to {url}, status code {response.status_code}"
))
return None
person_id = response.json()["_links"]["osdi:person"]["href"].split(
"/")[-1]
# if (first) subscriber field not set, set it
if slug and not response.json().get("custom_fields",
{}).get("first_subscriber"):
response = session.put(
PEOPLE_ENDPOINT.format(person_id=person_id),
json={
"custom_fields": {
"subscriber": slug,
},
},
)
except Exception as e:
extra = {"url": url, "info": info, "exception": str(e)}
logger.error(
"actionnetwork: Error posting to %(url)s, info %(info)s, exception %(exception)s",
extra,
extra=extra,
)
sentry_sdk.capture_exception(
ActionNetworkError(f"Error posting to {url}, exception {str(e)}"))
return None
# return link to the *person*, not the submission
return person_id
def geocode(**kwargs):
RETRIES = 2
TIMEOUT = 6.0 # seems to be enough to handle slow apartment build queries
args = {}
for k in ["street", "city", "state", "q", "fields"]:
if k in kwargs:
if k == "street" and kwargs[k]:
args[k] = strip_address_number_alpha_suffix(kwargs[k])
else:
args[k] = kwargs[k]
if "zipcode" in kwargs:
args["postal_code"] = kwargs["zipcode"]
url = f"{API_ENDPOINT}?{urlencode({**args, 'api_key': settings.GEOCODIO_KEY})}"
with statsd.timed("turnout.common.geocode.geocode", sample_rate=0.2):
retries = Retry(total=RETRIES,
backoff_factor=1,
status_forcelist=[429, 500, 502, 503, 504])
http = requests.Session()
http.mount("https://", HTTPAdapter(max_retries=retries))
try:
with tracer.trace("geocode", service="geocodioclient"):
r = http.get(url, timeout=TIMEOUT)
except Exception as e:
extra = {
"url": API_ENDPOINT,
"api_args": str(args),
"exception": str(e)
}
logger.warning(
"Error querying geocodio args %(api_args)s, exception %(exception)s",
extra,
extra=extra,
)
sentry_sdk.capture_exception(
GeocodioAPIError(
f"Error querying {API_ENDPOINT}, exception {str(e)}"))
return None
if r.status_code != 200:
extra = {
"url": API_ENDPOINT,
"api_args": str(args),
"status_code": r.status_code,
}
logger.warning(
"Error querying geocodio args %(api_args)s, status code %(status_code)s",
extra,
extra=extra,
)
if r.status_code != 422: # we get this from bogus addresses
sentry_sdk.capture_exception(
GeocodioAPIError(
f"Error querying {API_ENDPOINT}, status code {r.status_code}"
))
return None
return r.json().get("results", None)
def get_or_create_lob_address(
internal_id: str,
name: str,
address1: str,
address2: Optional[str],
city: str,
state: str,
zipcode: str,
country: str = "US",
) -> str:
# check cache
addr = cache.get(f"lob_addr_{internal_id}", None)
if addr:
return addr
# lookup
with tracer.trace("lob.address.list", service="lob"):
for addr in lob.Address.list(metadata={"va_id": internal_id})["data"]:
if (
addr["name"] == name.upper()
and addr["address_line1"] == address1.upper()
and (addr["address_line2"] or "") == (address2 or "").upper()
and addr["address_city"] == city.upper()
and addr["address_state"] == state.upper()
and addr["address_zip"][0 : len(zipcode)] == zipcode
):
cache.set(f"lob_addr_{internal_id}", addr["id"])
return addr["id"]
# create
with tracer.trace("lob.address.create", service="lob"):
r = lob.Address.create(
name=name,
address_line1=address1,
address_line2=address2,
address_city=city,
address_state=state,
address_zip=zipcode,
address_country=country,
metadata={"va_id": internal_id},
)
cache.set(f"lob_addr_{internal_id}", r.id)
return r.id
def get_alloy_freshness():
freshness = cache.get("alloy_freshness", None)
if not freshness:
session = get_session()
with tracer.trace("alloy.freshness", service="alloyapi"):
response = session.get(ALLOY_FRESHNESS_ENDPOINT, )
freshness = response.json().get("data", {}).get("data_freshness", {})
if freshness:
cache.set("alloy_freshness", freshness)
return freshness
def verify_address(
address1: str, address2: Optional[str], city: str, state: str, zipcode: str
) -> Tuple[bool, Dict[str, Any]]:
with tracer.trace("lob.verify_address", service="lob"):
r = lob.USVerification.create(
primary_line=address1,
secondary_line=address2,
city=city,
state=state,
zip_code=zipcode,
)
return r["deliverability"] == "deliverable", r["components"]
def purge_cdn_tags(tags: Iterable[str]) -> None:
final_tags = [generate_scoped_tag(tag) for tag in tags]
extra = {
"tags": final_tags,
}
logger.info("Cache Tag Purge %(tags)s", extra, extra=extra)
if not settings.CLOUDFLARE_ENABLED:
logger.info("Cloudflare Disabled")
return
with tracer.trace("cf.purge_cache", service="cloudflareclient"):
cf = CloudFlare.CloudFlare(token=settings.CLOUDFLARE_TOKEN)
cf.zones.purge_cache.delete(identifier1=settings.CLOUDFLARE_ZONE,
data={"tags": final_tags})
def resubscribe_phone(phone):
# re-subscribe the most recent user of this phone number
person_id = lookup_person_by_phone(phone)
if person_id:
session = get_session(settings.ACTIONNETWORK_KEY)
with tracer.trace("an.person_update", service="actionnetwork"):
response = session.put(
PEOPLE_ENDPOINT.format(person_id=person_id),
json={
"phone_numbers": [{
"number": str(phone),
"status": "subscribed"
}]
},
)
logger.info(
f"Resubscribed {phone} to actionnetwork person_id {person_id}")
return True
else:
return False
def send_sendgrid_mail(key):
mail = cache.get(key)
try:
with tracer.trace("sg.mail.send", service="sendgridclient"):
sg.client.mail.send.post(request_body=mail)
except Exception:
statsd.increment("turnout.mailer.sendgrid_send_task_server_error")
raise
finally:
cache.delete(key)
extra = {
"subject": mail["subject"],
"sent_from": mail["from"]["email"],
"sent_to": [d["email"] for d in mail["personalizations"][0]["to"]],
}
logger.info("Email Sent %(subject)s from %(sent_from)s to %(sent_to)s",
extra,
extra=extra)
logger.debug(mail)
def send_sms(self, text, ignore_opt_out=False, blast=None, **kwargs):
if self.opt_out_time and not ignore_opt_out:
return
if not twilio_client:
logger.warning(f"No twilio credentials, not sending sms: {text}")
return
with tracer.trace("smsbot.number.send_sms", service="twilio"):
msg = SMSMessage.objects.create(
phone=self,
direction=MessageDirectionType.OUT,
message=text,
blast=blast,
kwargs=kwargs,
)
max_tries = 2
tries = 0
while True:
tries += 1
if tries > max_tries:
logger.warning(
f"Gave up sending to twilio after {max_tries} tries: {msg}"
)
break
try:
r = twilio_client.messages.create(
to=str(self.phone),
messaging_service_sid=settings.
TWILIO_MESSAGING_SERVICE_SID,
body=text,
status_callback=msg.delivery_status_webhook(),
**kwargs,
)
msg.twilio_sid = r.sid
msg.save()
break
except ConnectionError as e:
logger.info(
f"Failed to send via twilio (attempt {tries}): {e} ({msg})"
)
time.sleep(tries)
def get_form(session, form_description: str, form_name: str) -> str:
if settings.ENV != "prod":
form_name = f"staging_{form_name}"
cache_key = f"actionnetwork_form_{form_name}"
form_id = cache.get(cache_key)
if form_id:
return form_id
nexturl = FORM_ENDPOINT
while nexturl:
with tracer.trace("an.form", service="actionnetwork"):
response = session.get(nexturl, )
for form in response.json()["_embedded"]["osdi:forms"]:
an_id = get_form_id(form["identifiers"], form_name)
if an_id:
logger.info(f"Found existing {form_name} form {an_id}")
cache.set(cache_key, an_id)
return an_id
nexturl = response.json().get("_links", {}).get("next", {}).get("href")
raise ActionNetworkError(f"Missing form {form_name}")
def get_or_create_form(session: requests.Session) -> str:
logger.info(f"Fetching forms from ActionNetwork")
form_name = f"{settings.ENV}_{settings.MOVER_SOURCE}_lead"
cache_key = f"actionnetwork_form_{form_name}"
form_id = cache.get(cache_key)
if form_id:
return form_id
nexturl = ACTIONNETWORK_FORM_ENDPOINT
while nexturl:
with tracer.trace("an.form", service="actionnetwork"):
response = session.get(nexturl, )
for form in response.json()["_embedded"]["osdi:forms"]:
an_id = get_form_id(form["identifiers"], form_name)
if an_id:
logger.info(f"Found existing {form_name} form {an_id}")
cache.set(cache_key, an_id)
return an_id
nexturl = response.json().get("_links", {}).get("next", {}).get("href")
raise ActionNetworkError(f"Missing form {form_name}")
def sync_subscriber(subscriber: Client) -> None:
session = get_session(settings.ACTIONNETWORK_SUBSCRIBERS_KEY)
form_id = get_form(session, "VoteAmerica Tool Users", "subscriber")
if hasattr(subscriber, "subscription"):
subscription = subscriber.subscription
else:
logger.info(f"No Subscription for subscriber {subscriber}")
return
info = {
"person": {
"given_name": subscription.primary_contact_first_name,
"family_name": subscription.primary_contact_last_name,
"email_addresses": [{
"address": subscription.primary_contact_email
}],
"custom_fields": {
"subscriber_is_c3":
subscription.plan == SubscriberPlan.NONPROFIT,
"subscriber_slug": subscriber.default_slug.slug,
},
},
}
url = ADD_ENDPOINT.format(form_id=form_id)
with tracer.trace("an.subscriber_form", service="actionnetwork"):
response = session.post(url, json=info)
if response.status_code != 200:
sentry_sdk.capture_exception(
ActionNetworkError(
f"Error posting subscriber to form {url}, status code {response.status_code}"
))
logger.warning(
f"Failed to post subscriber {subscriber} info {info} to actionnetwork: {response.text}"
)
logger.info(f"Posted subscriber {subscriber} to actionnetwork")
def submit_lob(
description: str,
action: Action,
to_addr,
item_file,
subscriber,
double_sided: bool = False,
) -> datetime.datetime:
# Try to only do it once. (This is a racy check, though!)
link = (
Link.objects.filter(action=action, external_tool=enums.ExternalToolType.LOB)
.order_by("created_at")
.first()
)
if link:
logger.info(
f"Already submitted lob letter {link.external_id} for {description}"
)
return link.created_at
from_addr = get_or_create_lob_address(
"va_return_addr",
settings.RETURN_ADDRESS["name"],
settings.RETURN_ADDRESS["address1"],
None,
settings.RETURN_ADDRESS["city"],
settings.RETURN_ADDRESS["state"],
settings.RETURN_ADDRESS["zipcode"],
)
with tracer.trace("lob.letter.create", service="lob"):
letter = lob.Letter.create(
description=description,
to_address=to_addr,
from_address=from_addr,
file=item_file,
color=False,
double_sided=double_sided,
address_placement="top_first_page",
return_envelope=RETURN_ENVELOPE,
perforated_page=COVER_SHEET_PERFORATED_PAGE,
metadata={"action_uuid": action.uuid},
)
link = Link.objects.create(
action=action,
subscriber=subscriber,
external_tool=enums.ExternalToolType.LOB,
external_id=letter["id"],
)
# make sure I'm the first and only one who completed
first = (
Link.objects.filter(action=action, external_tool=enums.ExternalToolType.LOB)
.order_by("created_at", "uuid")
.first()
)
if first != link:
logger.info(
f"Submitted a duplicate letter to lob ({link.external_id} != first {first.external_id}); canceling"
)
lob.Letter.delete(letter["id"])
link.delete()
return first.created_at
logger.info(f"Submitted lob letter for {description}")
return link.created_at
def _push_lead(session: requests.Session, form_id: str,
lead: MoverLead) -> None:
info = {
"person": {
"given_name":
lead.first_name,
"family_name":
lead.last_name,
"email_addresses": [{
"address": lead.email
}],
"postal_addresses": [
{
"address_lines":
[lead.new_address1, lead.new_address2 or ""],
"locality": lead.new_city,
"region": lead.new_state,
"postal_code": lead.new_zipcode,
"country": "US",
},
],
"custom_fields": {
f"{settings.MOVER_SOURCE}_new_address":
" ".join([lead.new_address1, lead.new_address2 or ""]),
f"{settings.MOVER_SOURCE}_new_city":
lead.new_city,
f"{settings.MOVER_SOURCE}_new_state":
lead.new_state,
f"{settings.MOVER_SOURCE}_new_zipcode":
lead.new_zipcode,
f"{settings.MOVER_SOURCE}_new_housing_tenure":
lead.new_housing_tenure,
f"{settings.MOVER_SOURCE}_old_address":
" ".join([lead.old_address1, lead.old_address2 or ""]),
f"{settings.MOVER_SOURCE}_old_city":
lead.old_city,
f"{settings.MOVER_SOURCE}_old_state":
lead.old_state,
f"{settings.MOVER_SOURCE}_old_zipcode":
lead.old_zipcode,
f"{settings.MOVER_SOURCE}_old_housing_tenure":
lead.old_housing_tenure,
f"{settings.MOVER_SOURCE}_move_date":
lead.move_date.isoformat(),
f"{settings.MOVER_SOURCE}_cross_state":
lead.old_state != lead.new_state,
f"{settings.MOVER_SOURCE}_cross_region":
lead.old_region_id != lead.new_region_id,
},
},
}
url = ACTIONNETWORK_ADD_ENDPOINT.format(form_id=form_id)
with tracer.trace("an.mover_form", service="actionnetwork"):
response = session.post(
url,
json=info,
)
if response.status_code != 200:
sentry_sdk.capture_exception(
ActionNetworkError(
f"Error posting mover lead to form {url}, status code {response.status_code}"
))
logger.error(
f"Failed to post {lead} info {info} to actionnetwork: {response.text}"
)
else:
person_id = response.json()["_links"]["osdi:person"]["href"].split(
"/")[-1]
logger.info(f"Synced {lead} to actionnetwork person {person_id}")
lead.actionnetwork_person_id = person_id
lead.save()
def ts_to_region(street=None,
city=None,
state=None,
zipcode=None,
latitude=None,
longitude=None):
if latitude and longitude:
args = {
"search_type": "point",
"latitude": latitude,
"longitude": longitude,
}
else:
args = {
"search_type": "address",
"address": f"{street}, {city}, {state} {zipcode}",
"state": state,
"zip5": zipcode,
}
url = f"{TARGETSMART_DISTRICT_API}?{urlencode({**args})}"
with tracer.trace("ts.district", service="targetsmartapi"):
r = requests.get(url, headers={"x-api-key": settings.TARGETSMART_KEY})
if r.status_code != 200:
extra = {"url": url, "status_code": r.status_code}
logger.error(
"Error querying TS district %{url}s, status code %{status_code}s",
extra,
extra=extra,
)
sentry_sdk.capture_exception(
TargetSmartAPIError(
f"Error querying {url}, status code {r.status_code}"))
return None
try:
match_data = r.json()["match_data"]
if not match_data:
return None
state_code = match_data["vb.vf_reg_cass_state"]
if state_code != state:
# if the address match gets the state wrong, return *no* result.
logger.warning(
f"User-provided state {state} does not match geocoded state in {match_data}"
)
return None
county = match_data["vb.vf_county_name"].lower()
city = (match_data["vb.vf_municipal_district"].lower()
or match_data["vb.vf_township"].lower())
except KeyError:
extra = {"url": url, "response": r.json()}
logger.error(
"Malformed result from TS district %(url)s, response %(response)s",
extra,
extra=extra,
)
sentry_sdk.capture_exception(
TargetSmartAPIError(
f"Malformed response from {url}, response {r.json()}"))
return None
queryset = Region.visible.filter(state__code=state_code).order_by("name")
if state_code in region_name_contains_county + county_method:
queryset = queryset.filter(name__icontains=county)
ls = list(queryset)
regions_by_name = {r.name.lower(): r for r in ls}
if state_code == "WI":
# land o' lakes
city = city.replace("o'", "o' ")
if state_code == "AL" and county == "Jefferson":
# warning: I think this isn't super precise since the lat/lng
# comes from the zip?
location = Point(
r.json()["match_data"]["z9_longitude"],
r.json()["match_data"]["z9_latitude"],
)
if al_jefferson_county_bessemer_division(location):
return [regions_by_name["jefferson county, bessemer division"]]
else:
return [regions_by_name["jefferson county, birmingham division"]]
if state_code == "NY":
key = nyc_remap.get(county, None)
if key and key in regions_by_name:
return [regions_by_name[key]]
if state_code == "MO":
if city in ["kansas city", "dist kansas city"]:
return [regions_by_name["kansas city"]]
if county == "st louis city":
return [regions_by_name["saint louis city"]]
if county == "st louis":
return [regions_by_name["saint louis county"]]
if state_code == "IL":
if county == "city of east st louis":
county = "city of east saint louis"
for fmt in [
# generic city matches
"City of {city}",
"Town of {city}",
"{city} Township",
"{city}, {county} County",
"{city}",
# michigan-specific (try local office *before* county office--mail will get processed faster)
"{city}, {county} County",
"{city} Township, {county}",
"{city} Township, {county} County",
"{city} City, {county}",
"{city} City, {county} County",
# wi
"City of {city}, {county} County",
"Town of {city}, {county} County",
"Village of {city}, {county} County",
# generic county
"{county}",
"{county} County",
]:
key = fmt.format(city=city, county=county).lower()
if key in regions_by_name:
return [regions_by_name[key]]
return []
def setup_action_forms(subscriber_id, api_key, slug):
if subscriber_id:
key = f"{CACHE_KEY}-{subscriber_id}"
else:
key = CACHE_KEY
forms = cache.get(key) or {}
missing = False
for action in ACTIONS.keys():
if action not in forms:
missing = True
if not missing:
return forms
session = get_session(api_key)
logger.info(
f"Fetching forms from ActionNetwork for subscriber {subscriber_id}")
prefix = settings.ACTIONNETWORK_FORM_PREFIX
forms = {}
with tracer.trace("an.form", service="actionnetwork"):
nexturl = FORM_ENDPOINT
while nexturl:
logger.info(nexturl)
response = session.get(nexturl)
for form in response.json()["_embedded"]["osdi:forms"]:
an_id, va_action = get_form_ids(form["identifiers"], prefix)
if an_id and va_action:
forms[va_action] = an_id
if va_action in ACTIONS:
title = get_form_title(ACTIONS[va_action], prefix)
if title != form["title"]:
logger.info(
f"Fixing title for {va_action} form {an_id}")
response = session.put(
FORM_ENDPOINT + f"/{an_id}",
json={
"title": title,
},
)
nexturl = response.json().get("_links", {}).get("next",
{}).get("href")
for action, tool in ACTIONS.items():
if action not in forms:
# This code runs once per action, ever.
logger.info(f"Creating action form for {tool} ({prefix})")
with tracer.trace("an.form.create", service="actionnetwork"):
form_id = f"voteamerica:{prefix}_{action}"
if subscriber_id and slug:
form_id += "_" + slug
response = session.post(
FORM_ENDPOINT,
json={
"identifiers": [form_id],
"title": get_form_title(tool, prefix),
"origin_system": "voteamerica",
},
)
logger.info(response.json())
an_id, va_action = get_form_ids(response.json()["identifiers"],
prefix)
if an_id and va_action:
forms[va_action] = an_id
cache.set(key, forms, settings.ACTIONNETWORK_FORM_CACHE_TIMEOUT)
return forms
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
state_code = serializer.validated_data["state"]
with tracer.trace("verifier.apicalls"):
alloy = gevent.spawn(query_alloy, serializer.validated_data)
targetsmart = gevent.spawn(query_targetsmart,
serializer.validated_data)
gevent.joinall([alloy, targetsmart])
serializer.validated_data["targetsmart_response"] = targetsmart.value
serializer.validated_data["alloy_response"] = alloy.value
if alloy.value.get("error"):
statsd.increment("turnout.verifier.alloy_error")
logger.error(f"Alloy Error {alloy.value['error']}")
if targetsmart.value.get("error"):
statsd.increment("turnout.verifier.ts_error")
logger.error(f"Targetsmart Error {targetsmart.value['error']}")
if targetsmart.value.get("error") and alloy.value.get("error"):
return Response(
{"error": "Error from data providers"},
status=status.HTTP_503_SERVICE_UNAVAILABLE,
)
alloy_record = alloy.value.get("data", {})
if len(targetsmart.value.get("result_set", [])) == 1:
targetsmart_record = targetsmart.value.get("result_set")[0]
else:
targetsmart_record = {}
targetsmart_status = enums.VoterStatus.UNKNOWN
if targetsmart_record.get("vb.vf_voter_status") == "Inactive":
targetsmart_status = enums.VoterStatus.INACTIVE
elif targetsmart_record.get("vb.vf_voter_status") == "Active":
targetsmart_status = enums.VoterStatus.ACTIVE
alloy_status = enums.VoterStatus.UNKNOWN
if alloy_record.get("registration_status") == "Inactive":
alloy_status = enums.VoterStatus.INACTIVE
elif alloy_record.get("registration_status") == "Active":
alloy_status = enums.VoterStatus.ACTIVE
registered = False
final_status = enums.VoterStatus.UNKNOWN
if (alloy_status == enums.VoterStatus.INACTIVE
or targetsmart_status == enums.VoterStatus.INACTIVE):
final_status = enums.VoterStatus.INACTIVE
elif (alloy_status == enums.VoterStatus.ACTIVE
or targetsmart_status == enums.VoterStatus.ACTIVE):
final_status = enums.VoterStatus.ACTIVE
registered = True
serializer.validated_data["targetsmart_status"] = targetsmart_status
serializer.validated_data["alloy_status"] = alloy_status
serializer.validated_data["voter_status"] = final_status
serializer.validated_data["registered"] = registered
if alloy_status != targetsmart_status:
statsd.increment("turnout.verifier.vendor_mismatch",
tags=[f"state:{state_code}"])
serializer.validated_data["state"] = State.objects.get(code=state_code)
instance = serializer.save()
instance.action.track_event(EventType.FINISH)
action_finish.delay(instance.action.pk)
response = {"registered": registered, "action_id": instance.action.pk}
if registered:
statsd.increment("turnout.verifier.registered",
tags=[f"state:{state_code}"])
else:
statsd.increment("turnout.verifier.unregistered",
tags=[f"state:{state_code}"])
return Response(response)
def send_map_mms(
number: Number,
map_type: str, # 'pp' or 'ev'
address_full: str,
content: str = None,
blast: Blast = None,
) -> Optional[str]:
formdata: Dict[str, str] = {}
# geocode home
home = geocode(q=address_full)
home_address = address_full
if not home:
logger.info(f"{number}: Failed to geocode {address_full}")
return f"Failed to geocode {address_full}"
formdata["home_address_short"] = address_full.split(",")[0].upper()
home_loc = f"{home[0]['location']['lng']},{home[0]['location']['lat']}"
# geocode destination
if get_feature_bool("locate_use_dnc_data",
home[0]["address_components"]["state"]):
# pollproxy
with tracer.trace("pollproxy.lookup", service="pollproxy"):
response = requests.get(DNC_API_ENDPOINT,
{"address": home_address})
if response.status_code != 200:
logger.warning(
f"Got {response.status_code} from dnc query on {home_address}")
return f"Failure querying data source"
if map_type == "pp":
dest = response.json().get("data",
{}).get("election_day_locations", [])
if not dest:
logger.info(
f"{number}: No election day location for {address_full}: {response.json()}"
)
return f"No election day location for {address_full}"
dest = dest[0]
elif map_type == "ev":
dest = response.json().get("data", {}).get("early_vote_locations",
[])
if not dest:
logger.info(
f"{number}: No early_vote location for {address_full}")
return f"No early vote location for {address_full}"
dest = dest[0]
else:
return f"Unrecognized address type {map_type}"
dest_name = dest["location_name"]
dest_address = (
f"{dest['address_line_1']}, {dest['city']}, {dest['state']} {dest['zip']}"
)
dest_hours = dest["dates_hours"]
if "lon" not in dest or "lat" not in dest:
logger.warning(
f"Lon/lat missing from {dest_address} (pollproxy): {dest}")
return f"Lon/lat missing from {dest_address} (pollproxy)"
dest_lon = dest["lon"]
dest_lat = dest["lat"]
else:
# civic
with tracer.trace("civicapi.voterlookup", service="google"):
response = requests.get(
CIVIC_API_ENDPOINT,
{
"address": home_address,
"electionId": 7000,
"key": settings.CIVIC_KEY,
},
)
if response.status_code != 200:
logger.warning(
f"Got {response.status_code} from civic query on {home_address}"
)
return f"Failure querying data source"
if map_type == "pp":
dest = response.json().get("pollingLocations", [])
if not dest:
logger.info(
f"{number}: No election day location for {address_full}: {dest}"
)
return f"No election day location for {address_full}"
dest = dest[0]
elif map_type == "ev":
dest = response.json().get("earlyVoteSites", [])
if not dest:
logger.info(
f"{number}: No early_vote location for {address_full}")
return f"No early vote location for {address_full}"
dest = dest[0]
else:
return f"Unrecognized address type {map_type}"
dest_name = dest["address"]["locationName"]
dest_address = f"{dest['address']['line1']}, {dest['address']['city']}, {dest['address']['state']} {dest['address']['zip']}"
dest_hours = dest["pollingHours"]
if "longitude" not in dest or "latitude" not in dest:
logger.warning(
f"Lon/lat missing from {dest_address} (pollproxy): {dest}")
return f"Lon/lat missing from {dest_address} (civic)"
dest_lon = dest["longitude"]
dest_lat = dest["latitude"]
formdata["dest_name"] = dest_name.upper()
formdata["dest_address"] = dest_address.upper()
formdata["dest_hours"] = dest_hours
# Pick a reasonable zoom level for the map, since mapbox pushes
# the markers to the very edge of the map.
#
# mapbox zoom levels are 1-20, and go by power of 2: +1 zoom means
# 1/4 of the map area.
dx = abs(float(dest_lon) - float(home[0]["location"]["lng"]))
dy = abs(float(dest_lat) - float(home[0]["location"]["lat"]))
if dx > dy:
d = dx
else:
d = dy
logd = math.log2(1.0 / d)
zoom = logd + 7.5
if HIRES:
size = "512x512"
else:
size = "256x256"
zoom -= 1.0
centerx = (float(dest_lon) + float(home[0]["location"]["lng"])) / 2
centery = (float(dest_lat) + float(home[0]["location"]["lat"])) / 2
map_loc = f"{centerx},{centery},{zoom}"
# fetch map
map_url = f"https://api.mapbox.com/styles/v1/mapbox/streets-v11/static/pin-s-home+00f({home_loc}),pin-s-p+f00({dest_lon},{dest_lat})/{map_loc}/{size}?access_token={settings.MAPBOX_KEY}"
response = requests.get(map_url)
if response.status_code != 200:
logger.warning(
f"{number}: Failed to fetch map, got status code {response.status_code}"
)
return "Failed to fetch map"
map_image = response.content
map_image_type = response.headers["content-type"]
# store map in s3
filename = str(uuid.uuid4()) + "." + map_image_type.split("/")[-1]
upload = s3_client.put_object(
Bucket=settings.MMS_ATTACHMENT_BUCKET,
Key=filename,
ContentType=map_image_type,
ACL="public-read",
Body=map_image,
)
if upload.get("ResponseMetadata", {}).get("HTTPStatusCode") != 200:
logger.warning(
f"{number}: Unable to push {filename} to {settings.MMS_ATTACHMENT_BUCKET}"
)
return "Unable to upload map"
stored_map_url = (
f"https://{settings.MMS_ATTACHMENT_BUCKET}.s3.amazonaws.com/{filename}"
)
# locator link
locator_url = f"https://www.voteamerica.com/where-to-vote/?{urlencode({'address':home_address})}"
if blast:
locator_url += f"&utm_medium=mms&utm_source=turnout&utm_campaign={blast.campaign}&source=va_mms_turnout_{blast.campaign}"
formdata["locator_url"] = shorten_url(locator_url)
# send
number.send_sms(
content.format(**formdata),
media_url=[stored_map_url],
blast=blast,
)
logger.info(
f"Sent {map_type} map for {address_full} (blast {blast}) to {number}")
return None
