def load_ehpa_signings(self):
print("Processing EHPA signings.")
writer = BatchWriter(models.EmergencyHPASigning)
signings = DocusignEnvelope.objects.filter(
status=HP_DOCUSIGN_STATUS_CHOICES.SIGNED, )
with writer.atomic_transaction(using=settings.DWH_DATABASE, wipe=True):
for signing in signings:
writer.write(
models.EmergencyHPASigning(created_at=signing.created_at))
class AnalyticsLogger:
BATCH_SIZE = 1000
def __init__(self, client: TembaClient):
self.client = client
self.writer = BatchWriter(models.RapidproRun)
def log_run(
self,
flow: Flow,
run: Run,
num_error_steps: Optional[int] = None,
was_rent_history_received: Optional[bool] = None,
):
run = models.RapidproRun(
flow_uuid=flow.uuid,
flow_name=flow.name,
user_uuid=run.contact.uuid,
start_time=run.created_on,
end_time=run.exited_on,
num_steps=len(run.path),
exit_type=run.exit_type,
num_error_steps=num_error_steps,
was_rent_history_received=was_rent_history_received,
)
self.writer.write(run)
def process_rh_requests(self, flow: Flow, error_nodes=List[NodeDesc]):
error_uuids = flow.find_all_node_uuids(error_nodes)
for run in flow.iter_exited_runs(self.client):
errors = 0
for step in run.path:
if step.node in error_uuids:
errors += 1
self.log_run(flow, run, num_error_steps=errors)
def process_rh_followups(self,
flow: Flow,
yes_nodes=NodeDesc,
no_nodes=NodeDesc):
yes_uuids = flow.find_node_uuids(yes_nodes)
no_uuids = flow.find_node_uuids(no_nodes)
for run in flow.iter_exited_runs(self.client):
rh_received: Optional[bool] = None
for step in run.path:
if step.node in yes_uuids:
assert rh_received is None or rh_received is True
rh_received = True
elif step.node in no_uuids:
assert rh_received is None or rh_received is False
rh_received = False
self.log_run(flow, run, was_rent_history_received=rh_received)
def load_loc_requests(self):
print("Processing letter of complaint requests.")
writer = BatchWriter(models.LetterOfComplaintRequest)
kwargs = {
f"{name}_uuid": uuid_from_url(url)
for (name, url) in LOC_GROUP_URLS.items()
}
with connection.cursor() as cursor:
cursor.execute(LOC_SQLFILE.read_text(), kwargs)
with writer.atomic_transaction(using=settings.DWH_DATABASE,
wipe=True):
for row_dict in iter_cursor_dicts(cursor):
req = models.LetterOfComplaintRequest(**row_dict)
writer.write(req)
def load_online_rent_history_requests(self):
print("Processing online rent history requests.")
writer = BatchWriter(models.OnlineRentHistoryRequest)
with connection.cursor() as cursor:
cursor.execute("""
SELECT rh.created_at, rapidpro_contact.uuid AS user_uuid
FROM rh_rentalhistoryrequest AS rh
LEFT JOIN rapidpro_contact ON rh.phone_number = rapidpro_contact.phone_number
""")
with writer.atomic_transaction(using=settings.DWH_DATABASE,
wipe=True):
for row_dict in iter_cursor_dicts(cursor):
req = models.OnlineRentHistoryRequest(**row_dict)
writer.write(req)
def update_texting_history(
backfill: bool = False,
max_age: Optional[int] = None,
silent: bool = False,
) -> Optional[datetime.datetime]:
if not twilio.is_enabled():
return None
max_age = max_age or 99_999
client = twilio.get_client()
our_number = tendigit_to_e164(settings.TWILIO_PHONE_NUMBER)
earliest_from_us, latest_from_us = get_min_max_date_sent(
Message.objects.filter(is_from_us=True, our_phone_number=our_number))
earliest_to_us, latest_to_us = get_min_max_date_sent(
Message.objects.filter(is_from_us=False, our_phone_number=our_number))
max_age_date = now() - datetime.timedelta(days=max_age)
# The way Twilio's Python client retrieves messages is a bit confusing at first,
# but the documentation on their underlying REST API helps a bit:
#
# https://www.twilio.com/docs/sms/api/message-resource
#
# In short, it's not possible to provide *both* a maximum and a minimum date,
# and the results are always sorted in reverse chronological order, so we need
# to deal with that.
if backfill:
to_us_kwargs = {"date_sent_before": earliest_to_us}
from_us_kwargs = {"date_sent_before": earliest_from_us}
else:
to_us_kwargs = {"date_sent_after": latest_to_us}
from_us_kwargs = {"date_sent_after": latest_from_us}
all_messages = itertools.chain(
stop_when_older_than(
client.messages.stream(to=our_number, **to_us_kwargs),
max_age_date,
),
stop_when_older_than(
client.messages.stream(from_=our_number, **from_us_kwargs),
max_age_date,
),
)
with BatchWriter(Message, ignore_conflicts=True, silent=silent) as writer:
for sms in all_messages:
is_from_us = sms.from_ == our_number
model = Message(
sid=sms.sid,
ordering=get_ordering_for_sms(sms, is_from_us),
direction=sms.direction,
is_from_us=is_from_us,
user_phone_number=sms.to if is_from_us else sms.from_,
body=clean_body(sms.body),
status=sms.status,
date_created=sms.date_created,
date_sent=sms.date_sent,
date_updated=sms.date_updated,
error_code=sms.error_code,
error_message=sms.error_message,
our_phone_number=our_number,
)
if not silent:
print(sms.sid, sms.date_sent, sms.direction, sms.status)
model.clean_fields()
writer.write(model)
return Message.objects.all().aggregate(Max("date_sent"))["date_sent__max"]
def __init__(self, client: TembaClient):
self.client = client
self.writer = BatchWriter(models.RapidproRun)