def load_ehpa_signings(self): print("Processing EHPA signings.") writer = BatchWriter(models.EmergencyHPASigning) signings = DocusignEnvelope.objects.filter( status=HP_DOCUSIGN_STATUS_CHOICES.SIGNED, ) with writer.atomic_transaction(using=settings.DWH_DATABASE, wipe=True): for signing in signings: writer.write( models.EmergencyHPASigning(created_at=signing.created_at))
class AnalyticsLogger: BATCH_SIZE = 1000 def __init__(self, client: TembaClient): self.client = client self.writer = BatchWriter(models.RapidproRun) def log_run( self, flow: Flow, run: Run, num_error_steps: Optional[int] = None, was_rent_history_received: Optional[bool] = None, ): run = models.RapidproRun( flow_uuid=flow.uuid, flow_name=flow.name, user_uuid=run.contact.uuid, start_time=run.created_on, end_time=run.exited_on, num_steps=len(run.path), exit_type=run.exit_type, num_error_steps=num_error_steps, was_rent_history_received=was_rent_history_received, ) self.writer.write(run) def process_rh_requests(self, flow: Flow, error_nodes=List[NodeDesc]): error_uuids = flow.find_all_node_uuids(error_nodes) for run in flow.iter_exited_runs(self.client): errors = 0 for step in run.path: if step.node in error_uuids: errors += 1 self.log_run(flow, run, num_error_steps=errors) def process_rh_followups(self, flow: Flow, yes_nodes=NodeDesc, no_nodes=NodeDesc): yes_uuids = flow.find_node_uuids(yes_nodes) no_uuids = flow.find_node_uuids(no_nodes) for run in flow.iter_exited_runs(self.client): rh_received: Optional[bool] = None for step in run.path: if step.node in yes_uuids: assert rh_received is None or rh_received is True rh_received = True elif step.node in no_uuids: assert rh_received is None or rh_received is False rh_received = False self.log_run(flow, run, was_rent_history_received=rh_received)
def load_loc_requests(self): print("Processing letter of complaint requests.") writer = BatchWriter(models.LetterOfComplaintRequest) kwargs = { f"{name}_uuid": uuid_from_url(url) for (name, url) in LOC_GROUP_URLS.items() } with connection.cursor() as cursor: cursor.execute(LOC_SQLFILE.read_text(), kwargs) with writer.atomic_transaction(using=settings.DWH_DATABASE, wipe=True): for row_dict in iter_cursor_dicts(cursor): req = models.LetterOfComplaintRequest(**row_dict) writer.write(req)
def load_online_rent_history_requests(self): print("Processing online rent history requests.") writer = BatchWriter(models.OnlineRentHistoryRequest) with connection.cursor() as cursor: cursor.execute(""" SELECT rh.created_at, rapidpro_contact.uuid AS user_uuid FROM rh_rentalhistoryrequest AS rh LEFT JOIN rapidpro_contact ON rh.phone_number = rapidpro_contact.phone_number """) with writer.atomic_transaction(using=settings.DWH_DATABASE, wipe=True): for row_dict in iter_cursor_dicts(cursor): req = models.OnlineRentHistoryRequest(**row_dict) writer.write(req)
def update_texting_history( backfill: bool = False, max_age: Optional[int] = None, silent: bool = False, ) -> Optional[datetime.datetime]: if not twilio.is_enabled(): return None max_age = max_age or 99_999 client = twilio.get_client() our_number = tendigit_to_e164(settings.TWILIO_PHONE_NUMBER) earliest_from_us, latest_from_us = get_min_max_date_sent( Message.objects.filter(is_from_us=True, our_phone_number=our_number)) earliest_to_us, latest_to_us = get_min_max_date_sent( Message.objects.filter(is_from_us=False, our_phone_number=our_number)) max_age_date = now() - datetime.timedelta(days=max_age) # The way Twilio's Python client retrieves messages is a bit confusing at first, # but the documentation on their underlying REST API helps a bit: # # https://www.twilio.com/docs/sms/api/message-resource # # In short, it's not possible to provide *both* a maximum and a minimum date, # and the results are always sorted in reverse chronological order, so we need # to deal with that. if backfill: to_us_kwargs = {"date_sent_before": earliest_to_us} from_us_kwargs = {"date_sent_before": earliest_from_us} else: to_us_kwargs = {"date_sent_after": latest_to_us} from_us_kwargs = {"date_sent_after": latest_from_us} all_messages = itertools.chain( stop_when_older_than( client.messages.stream(to=our_number, **to_us_kwargs), max_age_date, ), stop_when_older_than( client.messages.stream(from_=our_number, **from_us_kwargs), max_age_date, ), ) with BatchWriter(Message, ignore_conflicts=True, silent=silent) as writer: for sms in all_messages: is_from_us = sms.from_ == our_number model = Message( sid=sms.sid, ordering=get_ordering_for_sms(sms, is_from_us), direction=sms.direction, is_from_us=is_from_us, user_phone_number=sms.to if is_from_us else sms.from_, body=clean_body(sms.body), status=sms.status, date_created=sms.date_created, date_sent=sms.date_sent, date_updated=sms.date_updated, error_code=sms.error_code, error_message=sms.error_message, our_phone_number=our_number, ) if not silent: print(sms.sid, sms.date_sent, sms.direction, sms.status) model.clean_fields() writer.write(model) return Message.objects.all().aggregate(Max("date_sent"))["date_sent__max"]
def __init__(self, client: TembaClient): self.client = client self.writer = BatchWriter(models.RapidproRun)