def do_export(self): """ Does actual export. Called from a celery task. """ book = Workbook() self.render_book(book) temp = NamedTemporaryFile(delete=True) book.save(temp) temp.flush() org_root = getattr(settings, 'SITE_ORGS_STORAGE_ROOT', 'orgs') filename = '%s/%d/%s/%s.xls' % (org_root, self.org_id, self.directory, random_string(20)) default_storage.save(filename, File(temp)) self.filename = filename self.save(update_fields=('filename',)) subject = "Your export is ready" download_url = self.org.make_absolute_url(reverse(self.download_view, args=[self.pk])) send_email([self.created_by], subject, 'utils/email/export', {'download_url': download_url}) # force a gc import gc gc.collect()
def _org_clean(org): super(Org, org).clean() # set config defaults if not org.config: org.config = json.dumps({ORG_CONFIG_SECRET_TOKEN: random_string(16).lower(), ORG_CONFIG_CHAT_NAME_FIELD: 'chat_name'})
def do_export(self): """ Does actual export. Called from a celery task. """ book = Workbook() search = self.get_search() book = self.render_book(book, search) temp = NamedTemporaryFile(delete=True) book.save(temp) temp.flush() filename = 'orgs/%d/%s/%s.xls' % (self.org_id, self.directory, random_string(20)) default_storage.save(filename, File(temp)) self.filename = filename self.save(update_fields=('filename',)) subject = "Your export is ready" host = settings.SITE_HOST_PATTERN % self.org.subdomain download_url = host + reverse(self.download_view, args=[self.pk]) send_email(self.created_by.username, subject, self.email_templates, dict(link=download_url)) # force a gc import gc gc.collect()
def create_org(self, name, timezone, subdomain): org = Org.objects.create( name=name, timezone=timezone, subdomain=subdomain, created_by=self.superuser, modified_by=self.superuser ) org.backends.get_or_create( api_token=random_string(32), slug="rapidpro", created_by=self.superuser, modified_by=self.superuser ) return org
def setUp(self): self.superuser = User.objects.create_superuser(username="******", email="*****@*****.**", password="******") self.admin = self.create_user("Administrator") self.anon = self.create_user("Anon") self.uganda = self.create_org("uganda", pytz.timezone("Africa/Kampala"), self.admin) self.nigeria = self.create_org("nigeria", pytz.timezone("Africa/Lagos"), self.admin) rapidpro_backend = self.nigeria.backends.filter(slug="rapidpro").first() if not rapidpro_backend: rapidpro_backend, created = self.nigeria.backends.get_or_create( api_token=random_string(32), slug="rapidpro", created_by=self.admin, modified_by=self.admin ) self.rapidpro_backend = rapidpro_backend floip_backend = self.nigeria.backends.filter(slug="floip").first() if not floip_backend: floip_backend, created = self.nigeria.backends.get_or_create( api_token=random_string(32), slug="floip", created_by=self.admin, modified_by=self.admin ) self.floip_backend = floip_backend
def do_export(self): """ Does actual export. Called from a celery task as it may require a lot of API calls to grab all messages. """ from xlwt import Workbook, XFStyle book = Workbook() date_style = XFStyle() date_style.num_format_str = 'DD-MM-YYYY HH:MM:SS' base_fields = ["Time", "Message ID", "Flagged", "Labels", "Text", "Contact"] contact_fields = self.org.get_contact_fields() all_fields = base_fields + contact_fields label_map = {l.name: l for l in Label.get_all(self.org)} client = self.org.get_temba_client() search = self.get_search() # fetch all messages to be exported messages = Message.search(self.org, search, None) # extract all unique contacts in those messages contact_uuids = set() for msg in messages: contact_uuids.add(msg.contact) # fetch all contacts in batches of 25 and organize by UUID contacts_by_uuid = {} for uuid_chunk in chunks(list(contact_uuids), 25): for contact in client.get_contacts(uuids=uuid_chunk): contacts_by_uuid[contact.uuid] = contact def add_sheet(num): sheet = book.add_sheet(unicode(_("Messages %d" % num))) for col in range(len(all_fields)): field = all_fields[col] sheet.write(0, col, unicode(field)) return sheet # even if there are no messages - still add a sheet if not messages: add_sheet(1) else: sheet_number = 1 for msg_chunk in chunks(messages, 65535): current_sheet = add_sheet(sheet_number) row = 1 for msg in msg_chunk: created_on = msg.created_on.astimezone(pytz.utc).replace(tzinfo=None) flagged = SYSTEM_LABEL_FLAGGED in msg.labels labels = ', '.join([label_map[l_name].name for l_name in msg.labels if l_name in label_map]) contact = contacts_by_uuid.get(msg.contact, None) # contact may no longer exist in RapidPro current_sheet.write(row, 0, created_on, date_style) current_sheet.write(row, 1, msg.id) current_sheet.write(row, 2, 'Yes' if flagged else 'No') current_sheet.write(row, 3, labels) current_sheet.write(row, 4, msg.text) current_sheet.write(row, 5, msg.contact) for cf in range(len(contact_fields)): if contact: contact_field = contact_fields[cf] current_sheet.write(row, 6 + cf, contact.fields.get(contact_field, None)) else: current_sheet.write(row, 6 + cf, None) row += 1 sheet_number += 1 temp = NamedTemporaryFile(delete=True) book.save(temp) temp.flush() filename = 'orgs/%d/message_exports/%s.xls' % (self.org.id, random_string(20)) default_storage.save(filename, File(temp)) self.filename = filename self.save(update_fields=('filename',)) subject = "Your messages export is ready" download_url = settings.SITE_HOST_PATTERN % self.org.subdomain + reverse('cases.messageexport_read', args=[self.pk]) send_email(self.created_by.username, subject, 'cases/email/message_export', dict(link=download_url)) # force a gc import gc gc.collect()
def do_export(self): """ Does actual export. Called from a celery task as it may require a lot of API calls to grab all messages. """ from xlwt import Workbook, XFStyle book = Workbook() date_style = XFStyle() date_style.num_format_str = 'DD-MM-YYYY HH:MM:SS' base_fields = [ "Time", "Message ID", "Flagged", "Labels", "Text", "Contact" ] contact_fields = self.org.get_contact_fields() all_fields = base_fields + contact_fields label_map = {l.name: l for l in Label.get_all(self.org)} client = self.org.get_temba_client() search = self.get_search() # fetch all messages to be exported messages = Message.search(self.org, search, None) # extract all unique contacts in those messages contact_uuids = set() for msg in messages: contact_uuids.add(msg.contact) # fetch all contacts in batches of 25 and organize by UUID contacts_by_uuid = {} for uuid_chunk in chunks(list(contact_uuids), 25): for contact in client.get_contacts(uuids=uuid_chunk): contacts_by_uuid[contact.uuid] = contact def add_sheet(num): sheet = book.add_sheet(unicode(_("Messages %d" % num))) for col in range(len(all_fields)): field = all_fields[col] sheet.write(0, col, unicode(field)) return sheet # even if there are no messages - still add a sheet if not messages: add_sheet(1) else: sheet_number = 1 for msg_chunk in chunks(messages, 65535): current_sheet = add_sheet(sheet_number) row = 1 for msg in msg_chunk: created_on = msg.created_on.astimezone( pytz.utc).replace(tzinfo=None) flagged = SYSTEM_LABEL_FLAGGED in msg.labels labels = ', '.join([ label_map[l_name].name for l_name in msg.labels if l_name in label_map ]) contact = contacts_by_uuid.get( msg.contact, None) # contact may no longer exist in RapidPro current_sheet.write(row, 0, created_on, date_style) current_sheet.write(row, 1, msg.id) current_sheet.write(row, 2, 'Yes' if flagged else 'No') current_sheet.write(row, 3, labels) current_sheet.write(row, 4, msg.text) current_sheet.write(row, 5, msg.contact) for cf in range(len(contact_fields)): if contact: contact_field = contact_fields[cf] current_sheet.write( row, 6 + cf, contact.fields.get(contact_field, None)) else: current_sheet.write(row, 6 + cf, None) row += 1 sheet_number += 1 temp = NamedTemporaryFile(delete=True) book.save(temp) temp.flush() filename = 'orgs/%d/message_exports/%s.xls' % (self.org.id, random_string(20)) default_storage.save(filename, File(temp)) self.filename = filename self.save(update_fields=('filename', )) subject = "Your messages export is ready" download_url = settings.SITE_HOST_PATTERN % self.org.subdomain + reverse( 'cases.messageexport_read', args=[self.pk]) send_email(self.created_by.username, subject, 'cases/email/message_export', dict(link=download_url)) # force a gc import gc gc.collect()
def create_org(self, name, timezone, subdomain): return Org.objects.create( name=name, timezone=timezone, subdomain=subdomain, api_token=random_string(32), created_by=self.superuser, modified_by=self.superuser)