def do_export(self):
"""
Does actual export. Called from a celery task.
"""
book = Workbook()
self.render_book(book)
temp = NamedTemporaryFile(delete=True)
book.save(temp)
temp.flush()
org_root = getattr(settings, 'SITE_ORGS_STORAGE_ROOT', 'orgs')
filename = '%s/%d/%s/%s.xls' % (org_root, self.org_id, self.directory, random_string(20))
default_storage.save(filename, File(temp))
self.filename = filename
self.save(update_fields=('filename',))
subject = "Your export is ready"
download_url = self.org.make_absolute_url(reverse(self.download_view, args=[self.pk]))
send_email([self.created_by], subject, 'utils/email/export', {'download_url': download_url})
# force a gc
import gc
gc.collect()
def _org_clean(org):
super(Org, org).clean()
# set config defaults
if not org.config:
org.config = json.dumps({ORG_CONFIG_SECRET_TOKEN: random_string(16).lower(),
ORG_CONFIG_CHAT_NAME_FIELD: 'chat_name'})
def do_export(self):
"""
Does actual export. Called from a celery task.
"""
book = Workbook()
search = self.get_search()
book = self.render_book(book, search)
temp = NamedTemporaryFile(delete=True)
book.save(temp)
temp.flush()
filename = 'orgs/%d/%s/%s.xls' % (self.org_id, self.directory, random_string(20))
default_storage.save(filename, File(temp))
self.filename = filename
self.save(update_fields=('filename',))
subject = "Your export is ready"
host = settings.SITE_HOST_PATTERN % self.org.subdomain
download_url = host + reverse(self.download_view, args=[self.pk])
send_email(self.created_by.username, subject, self.email_templates, dict(link=download_url))
# force a gc
import gc
gc.collect()
def create_org(self, name, timezone, subdomain):
org = Org.objects.create(
name=name, timezone=timezone, subdomain=subdomain, created_by=self.superuser, modified_by=self.superuser
)
org.backends.get_or_create(
api_token=random_string(32), slug="rapidpro", created_by=self.superuser, modified_by=self.superuser
)
return org
def setUp(self):
self.superuser = User.objects.create_superuser(username="******", email="*****@*****.**", password="******")
self.admin = self.create_user("Administrator")
self.anon = self.create_user("Anon")
self.uganda = self.create_org("uganda", pytz.timezone("Africa/Kampala"), self.admin)
self.nigeria = self.create_org("nigeria", pytz.timezone("Africa/Lagos"), self.admin)
rapidpro_backend = self.nigeria.backends.filter(slug="rapidpro").first()
if not rapidpro_backend:
rapidpro_backend, created = self.nigeria.backends.get_or_create(
api_token=random_string(32), slug="rapidpro", created_by=self.admin, modified_by=self.admin
)
self.rapidpro_backend = rapidpro_backend
floip_backend = self.nigeria.backends.filter(slug="floip").first()
if not floip_backend:
floip_backend, created = self.nigeria.backends.get_or_create(
api_token=random_string(32), slug="floip", created_by=self.admin, modified_by=self.admin
)
self.floip_backend = floip_backend
def do_export(self):
"""
Does actual export. Called from a celery task as it may require a lot of API calls to grab all messages.
"""
from xlwt import Workbook, XFStyle
book = Workbook()
date_style = XFStyle()
date_style.num_format_str = 'DD-MM-YYYY HH:MM:SS'
base_fields = ["Time", "Message ID", "Flagged", "Labels", "Text", "Contact"]
contact_fields = self.org.get_contact_fields()
all_fields = base_fields + contact_fields
label_map = {l.name: l for l in Label.get_all(self.org)}
client = self.org.get_temba_client()
search = self.get_search()
# fetch all messages to be exported
messages = Message.search(self.org, search, None)
# extract all unique contacts in those messages
contact_uuids = set()
for msg in messages:
contact_uuids.add(msg.contact)
# fetch all contacts in batches of 25 and organize by UUID
contacts_by_uuid = {}
for uuid_chunk in chunks(list(contact_uuids), 25):
for contact in client.get_contacts(uuids=uuid_chunk):
contacts_by_uuid[contact.uuid] = contact
def add_sheet(num):
sheet = book.add_sheet(unicode(_("Messages %d" % num)))
for col in range(len(all_fields)):
field = all_fields[col]
sheet.write(0, col, unicode(field))
return sheet
# even if there are no messages - still add a sheet
if not messages:
add_sheet(1)
else:
sheet_number = 1
for msg_chunk in chunks(messages, 65535):
current_sheet = add_sheet(sheet_number)
row = 1
for msg in msg_chunk:
created_on = msg.created_on.astimezone(pytz.utc).replace(tzinfo=None)
flagged = SYSTEM_LABEL_FLAGGED in msg.labels
labels = ', '.join([label_map[l_name].name for l_name in msg.labels if l_name in label_map])
contact = contacts_by_uuid.get(msg.contact, None) # contact may no longer exist in RapidPro
current_sheet.write(row, 0, created_on, date_style)
current_sheet.write(row, 1, msg.id)
current_sheet.write(row, 2, 'Yes' if flagged else 'No')
current_sheet.write(row, 3, labels)
current_sheet.write(row, 4, msg.text)
current_sheet.write(row, 5, msg.contact)
for cf in range(len(contact_fields)):
if contact:
contact_field = contact_fields[cf]
current_sheet.write(row, 6 + cf, contact.fields.get(contact_field, None))
else:
current_sheet.write(row, 6 + cf, None)
row += 1
sheet_number += 1
temp = NamedTemporaryFile(delete=True)
book.save(temp)
temp.flush()
filename = 'orgs/%d/message_exports/%s.xls' % (self.org.id, random_string(20))
default_storage.save(filename, File(temp))
self.filename = filename
self.save(update_fields=('filename',))
subject = "Your messages export is ready"
download_url = settings.SITE_HOST_PATTERN % self.org.subdomain + reverse('cases.messageexport_read', args=[self.pk])
send_email(self.created_by.username, subject, 'cases/email/message_export', dict(link=download_url))
# force a gc
import gc
gc.collect()
def do_export(self):
"""
Does actual export. Called from a celery task as it may require a lot of API calls to grab all messages.
"""
from xlwt import Workbook, XFStyle
book = Workbook()
date_style = XFStyle()
date_style.num_format_str = 'DD-MM-YYYY HH:MM:SS'
base_fields = [
"Time", "Message ID", "Flagged", "Labels", "Text", "Contact"
]
contact_fields = self.org.get_contact_fields()
all_fields = base_fields + contact_fields
label_map = {l.name: l for l in Label.get_all(self.org)}
client = self.org.get_temba_client()
search = self.get_search()
# fetch all messages to be exported
messages = Message.search(self.org, search, None)
# extract all unique contacts in those messages
contact_uuids = set()
for msg in messages:
contact_uuids.add(msg.contact)
# fetch all contacts in batches of 25 and organize by UUID
contacts_by_uuid = {}
for uuid_chunk in chunks(list(contact_uuids), 25):
for contact in client.get_contacts(uuids=uuid_chunk):
contacts_by_uuid[contact.uuid] = contact
def add_sheet(num):
sheet = book.add_sheet(unicode(_("Messages %d" % num)))
for col in range(len(all_fields)):
field = all_fields[col]
sheet.write(0, col, unicode(field))
return sheet
# even if there are no messages - still add a sheet
if not messages:
add_sheet(1)
else:
sheet_number = 1
for msg_chunk in chunks(messages, 65535):
current_sheet = add_sheet(sheet_number)
row = 1
for msg in msg_chunk:
created_on = msg.created_on.astimezone(
pytz.utc).replace(tzinfo=None)
flagged = SYSTEM_LABEL_FLAGGED in msg.labels
labels = ', '.join([
label_map[l_name].name for l_name in msg.labels
if l_name in label_map
])
contact = contacts_by_uuid.get(
msg.contact,
None) # contact may no longer exist in RapidPro
current_sheet.write(row, 0, created_on, date_style)
current_sheet.write(row, 1, msg.id)
current_sheet.write(row, 2, 'Yes' if flagged else 'No')
current_sheet.write(row, 3, labels)
current_sheet.write(row, 4, msg.text)
current_sheet.write(row, 5, msg.contact)
for cf in range(len(contact_fields)):
if contact:
contact_field = contact_fields[cf]
current_sheet.write(
row, 6 + cf,
contact.fields.get(contact_field, None))
else:
current_sheet.write(row, 6 + cf, None)
row += 1
sheet_number += 1
temp = NamedTemporaryFile(delete=True)
book.save(temp)
temp.flush()
filename = 'orgs/%d/message_exports/%s.xls' % (self.org.id,
random_string(20))
default_storage.save(filename, File(temp))
self.filename = filename
self.save(update_fields=('filename', ))
subject = "Your messages export is ready"
download_url = settings.SITE_HOST_PATTERN % self.org.subdomain + reverse(
'cases.messageexport_read', args=[self.pk])
send_email(self.created_by.username, subject,
'cases/email/message_export', dict(link=download_url))
# force a gc
import gc
gc.collect()
def create_org(self, name, timezone, subdomain):
return Org.objects.create(
name=name, timezone=timezone, subdomain=subdomain, api_token=random_string(32),
created_by=self.superuser, modified_by=self.superuser)
