def export_documents(export):
from contacts.imex import get_exportable_documents as contacts_documents
from invoicing.imex import get_exportable_documents as invoicing_documents
with respect_language(export.language):
archive_name = _('Vosae export.zip')
zipped = ContentFile('', archive_name)
f = zipfile.ZipFile(zipped, mode='w', compression=zipfile.ZIP_DEFLATED)
for documents, path_func, doc_func in contacts_documents(export):
for document in documents:
f.writestr(path_func(document), doc_func(document))
for documents, path_func, doc_func in invoicing_documents(export):
for document in documents:
f.writestr(path_func(document), doc_func(document))
f.close()
zipped.content_type = "application/zip"
export.zipfile = VosaeFile(
tenant=export.tenant,
uploaded_file=zipped,
issuer=export.issuer
)
export.zipfile.save()
export.update(set__zipfile=export.zipfile)
context = {
'tenant': export.tenant,
'file': export.zipfile,
'site': {'name': settings.SITE_NAME, 'url': settings.SITE_URL}
}
# Email to issuer
subject = _('Your Vosae export is available')
message = render_to_string('data_liberation/emails/export_finished.txt', context)
send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [export.issuer.email])
def create_report_photo(request):
'''This method is used to create citizens reports. Validation included.'''
#Test the submit content size (max 2MB)
if (int(request.META.get('CONTENT_LENGTH')) > 15000000):
return HttpResponseBadRequest(json.dumps({"error_key": "ERROR_REPORT_FILE_EXCEED_SIZE", "request":request.POST}),content_type='application/json')
data_report_id = request.POST.get('report_id')
# Check if the photo is base64 encoded
if request.POST.get('base64', False):
image_data_base64 = request.POST.get('report_file')
# Extract content and metadata from base64 info
metadata, file_base64 = image_data_base64.split(';base64,')
metadata = metadata.replace("data:", "")
# Extract name and type from metadata
data_file_type, data_file_name = metadata.split(',')
# Set (meta)data to file object
data_file_content = ContentFile(file_base64.decode('base64'), name=data_file_name)
data_file_content.content_type = data_file_type
else:
data_file_content = request.FILES.get('report_file')
#Verify that everything has been posted to create a citizen report.
if (data_report_id == None):
return HttpResponseBadRequest(json.dumps({"error_key": "ERROR_REPORT_FILE_MISSING_DATA_REPORT_ID", "request":request.POST}),content_type='application/json')
if (data_file_content == None):
return HttpResponseBadRequest(json.dumps({"error_key": "ERROR_REPORT_FILE_MISSING_DATA_REPORT_FILE", "request":request.POST}),content_type='application/json')
try:
#Retrieve the report
reference_report = Report.objects.get(id=data_report_id)
except Exception:
return HttpResponseBadRequest(json.dumps({"error_key": "ERROR_REPORT_FILE_NOT_FOUND", "request":request.POST}),content_type='application/json')
report_file = ReportFile()
try:
report_file.title = "Mobile Upload"
report_file.file = data_file_content
report_file.report = reference_report
report_file.file_creation_date = datetime.now()
# Set same creator than report creator
report_file.created_by = reference_report.citizen or reference_report.created_by
#Save given data
report_file.is_new_report = True
report_file.save()
except Exception:
return HttpResponseBadRequest(json.dumps({"error_key": "ERROR_REPORT_FILE_PROBLEM_DATA", "request":request.POST}),content_type='application/json')
#Return the report ID
return JsonHttpResponse({
'report_photo': report_file.id
})
def test_text_file(self):
"""
This method tests if content of the text file are extracted when
a text file is uploaded
"""
f = ContentFile('This is a sample content', name='test.txt')
f.content_type = 'text/plain'
content = extract_text_file(f)
self.assertEquals(content.strip(), 'This is a sample content')
def write_file_to_storage(file_object, content_type, path):
"""
write file_object to the default_storage at given path
"""
file_s3 = ContentFile(file_object)
file_s3.content_type = content_type
default_storage.save(path, file_s3)
return file_s3
def get_image(file_name='test.png', width=200, height=200):
image_file = StringIO()
image = Image.new('RGBA', size=(width, height), color=(255, 0, 255))
image.save(image_file, 'png')
image_file.seek(0)
the_file = ContentFile(image_file.read(), file_name)
the_file.content_type = 'image/png'
return the_file
def get_image(file_name='test.png', width=200, height=200):
image_file = StringIO()
image = Image.new('RGBA', size=(width, height), color=(255, 0, 255))
image.save(image_file, 'png')
image_file.seek(0)
the_file = ContentFile(image_file.read(), file_name)
the_file.content_type = 'image/png'
return the_file
def get_image(file_name="test.png", width=200, height=200):
image_file = StringIO()
image = Image.new("RGBA", size=(width, height), color=(255, 0, 255))
image.save(image_file, "png")
image_file.seek(0)
the_file = ContentFile(image_file.read(), file_name)
the_file.content_type = "image/png"
return the_file
def get_doc_document(file_name='document_2.doc'):
doc_file = File(
open(
normpath(
join(dirname(dirname(abspath(__file__))),
'files/document_2.doc')), 'rb'))
the_file = ContentFile(doc_file.read(), file_name)
the_file.content_type = 'application/msword'
return the_file
def get_pdf_document(file_name='document_1.pdf'):
pdf_file = File(
open(
normpath(
join(dirname(dirname(abspath(__file__))),
'files/document_1.pdf')), 'rb'))
the_file = ContentFile(pdf_file.read(), file_name)
the_file.content_type = 'application/pdf'
return the_file
def resized_image(path, folder):
image = imread(path, plugin='matplotlib')
width = 250
height = 334
dim = (width, height)
resized = cv2.resize(image, dim, interpolation=cv2.INTER_AREA)
color_correct = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
success, buffer = cv2.imencode(".jpg", color_correct)
new_image = ContentFile(buffer)
new_image.content_type = 'image/jpeg'
image_url = upload_image_file(new_image, folder)
return image_url
def test_create_not_supported(self):
xyz_file = StringIO()
xyz = Image.new('RGBA', size=(50, 50), color=(256, 0, 0))
xyz.save(xyz_file, 'png')
xyz_file.seek(0)
the_file = ContentFile(xyz_file.read(), 'test.xyz')
the_file.content_type = 'chemical/x-xyz'
MediaFile.objects.create(name='Test name',
description='Test Description',
contribution=ObservationFactory.create(),
creator=UserFactory.create(),
the_file=the_file)
def test_content_type(self):
"""
Test saving a file with a None content type.
"""
name = 'test_image.jpg'
content = ContentFile('data')
content.content_type = None
self.storage.save(name, content)
self.storage.bucket.Object.assert_called_once_with(name)
obj = self.storage.bucket.Object.return_value
obj.upload_fileobj.assert_called_with(content,
ExtraArgs={
'ContentType': 'image/jpeg',
})
def test_create_not_supported(self):
xyz_file = StringIO()
xyz = Image.new('RGBA', size=(50, 50), color=(256, 0, 0))
xyz.save(xyz_file, 'png')
xyz_file.seek(0)
the_file = ContentFile(xyz_file.read(), 'test.xyz')
the_file.content_type = 'chemical/x-xyz'
MediaFile.objects.create(
name='Test name',
description='Test Description',
contribution=ObservationFactory.create(),
creator=UserF.create(),
the_file=the_file
)
def export_documents(export):
from contacts.imex import get_exportable_documents as contacts_documents
from invoicing.imex import get_exportable_documents as invoicing_documents
with respect_language(export.language):
archive_name = _('Vosae export.zip')
zipped = ContentFile('', archive_name)
f = zipfile.ZipFile(zipped, mode='w', compression=zipfile.ZIP_DEFLATED)
for documents, path_func, doc_func in contacts_documents(export):
for document in documents:
f.writestr(path_func(document), doc_func(document))
for documents, path_func, doc_func in invoicing_documents(export):
for document in documents:
f.writestr(path_func(document), doc_func(document))
f.close()
zipped.content_type = "application/zip"
export.zipfile = VosaeFile(tenant=export.tenant,
uploaded_file=zipped,
issuer=export.issuer)
export.zipfile.save()
export.update(set__zipfile=export.zipfile)
context = {
'tenant': export.tenant,
'file': export.zipfile,
'site': {
'name': settings.SITE_NAME,
'url': settings.SITE_URL
}
}
# Email to issuer
plaintext_context = Context(
autoescape=False) # HTML escaping not appropriate in plaintext
subject = subject = _("Your Vosae export is available")
text_body = render_to_string(
'data_liberation/emails/export_finished.txt', context,
plaintext_context)
html_body = render_to_string(
"data_liberation/emails/export_finished.html", context)
message = EmailMultiAlternatives(
subject=subject,
from_email=settings.DEFAULT_FROM_EMAIL,
to=[export.issuer.email],
body=text_body)
message.attach_alternative(html_body, "text/html")
message.send()
def pic_storage_url(user, backend, url):
pic_file_name = '/pic/{}/{}'.format(backend, user)
# download cover image to cover_file
try:
r = requests.get(url)
pic_file = ContentFile(r.content)
content_type = r.headers.get('content-type', '')
if u'text' in content_type:
logger.warning('Cover return text for pic_url={}'.format(url))
return None
pic_file.content_type = content_type
default_storage.save(pic_file_name, pic_file)
return default_storage.url(pic_file_name)
except Exception, e:
# if there is a problem, return None for cover URL
logger.warning('Failed to store cover for username={}'.format(user))
return None
def test_content_type(self):
"""
Test saving a file with a None content type.
"""
name = 'test_image.jpg'
content = ContentFile('data')
content.content_type = None
self.storage.save(name, content)
self.storage.bucket.Object.assert_called_once_with(name)
obj = self.storage.bucket.Object.return_value
obj.upload_fileobj.assert_called_with(
content.file,
ExtraArgs={
'ContentType': 'image/jpeg',
'ACL': self.storage.default_acl,
}
)
def test_content_type(self):
"""
Test saving a file with a None content type.
"""
name = 'test_image.jpg'
content = ContentFile('data')
content.content_type = None
self.storage.save(name, content)
self.storage.bucket.get_key.assert_called_once_with(name)
key = self.storage.bucket.get_key.return_value
key.set_metadata.assert_called_with('Content-Type', 'image/jpeg')
key.set_contents_from_file.assert_called_with(
content,
headers={'Content-Type': 'image/jpeg'},
policy=self.storage.default_acl,
reduced_redundancy=self.storage.reduced_redundancy,
rewind=True)
def publish(self, page):
html = self.generate_html(page)
# publishes a resolved learning object to HTML and returns the storage
# path
file = ContentFile(html, name="index.html")
file.content_type = "text/html;charset=utf-8" # for S3
# set up upload path
path = page.get_absolute_url() + "index.html"
if path[0] == "/":
path = path[1:]
# upload
self.upload_replace_file(path, file)
# return storage path
return [
path,
]
def test_content_type(self):
"""
Test saving a file with a None content type.
"""
name = 'test_image.jpg'
content = ContentFile('data')
content.content_type = None
self.storage.save(name, content)
self.storage.bucket.get_key.assert_called_once_with(name)
key = self.storage.bucket.get_key.return_value
key.set_metadata.assert_called_with('Content-Type', 'image/jpeg')
key.set_contents_from_file.assert_called_with(
content,
headers={'Content-Type': 'image/jpeg'},
policy=self.storage.default_acl,
reduced_redundancy=self.storage.reduced_redundancy,
rewind=True
)
def _upload_instance(self, xml_file, instance_dir_path, files):
xml_doc = clean_and_parse_xml(xml_file.read())
xml = StringIO()
de_node = xml_doc.documentElement
for node in de_node.firstChild.childNodes:
xml.write(node.toxml())
new_xml_file = ContentFile(xml.getvalue())
new_xml_file.content_type = "text/xml"
xml.close()
attachments = []
for attach in de_node.getElementsByTagName("mediaFile"):
filename_node = attach.getElementsByTagName("filename")
filename = filename_node[0].childNodes[0].nodeValue
if filename in files:
file_obj = default_storage.open(os.path.join(instance_dir_path, filename))
mimetype, encoding = mimetypes.guess_type(file_obj.name)
media_obj = django_file(file_obj, "media_files[]", mimetype)
attachments.append(media_obj)
create_instance(self.user.username, new_xml_file, attachments)
def store_doab_cover(doab_id, redo=False):
"""
returns tuple: 1) cover URL, 2) whether newly created (boolean)
"""
cover_file_name = '/doab/%s/cover' % (doab_id)
# if we don't want to redo and the cover exists, return the URL of the cover
if not redo and default_storage.exists(cover_file_name):
return (default_storage.url(cover_file_name), False)
# download cover image to cover_file
url = "http://www.doabooks.org/doab?func=cover&rid={0}".format(doab_id)
try:
r = requests.get(url, allow_redirects=False) # requests doesn't handle ftp redirects.
if r.status_code == 302:
redirurl = r.headers['Location']
if redirurl.startswith(u'ftp'):
springerftp = SPRINGER_COVER.match(redirurl)
if springerftp:
redirurl = SPRINGER_IMAGE.format(springerftp.groups(1))
r = requests.get(redirurl)
else:
r = requests.get(url)
else:
r = requests.get(url)
cover_file = ContentFile(r.content)
content_type = r.headers.get('content-type', '')
if u'text/html' in content_type:
logger.warning('Cover return html for doab_id={}'.format(doab_id))
return (None, False)
cover_file.content_type = content_type
default_storage.save(cover_file_name, cover_file)
return (default_storage.url(cover_file_name), True)
except Exception as e:
# if there is a problem, return None for cover URL
logger.warning('Failed to make cover image for doab_id={}: {}'.format(doab_id, e))
return (None, False)
def _upload_instance(self, xml_file, instance_dir_path, files):
xml_doc = clean_and_parse_xml(xml_file.read())
xml = StringIO()
de_node = xml_doc.documentElement
for node in de_node.firstChild.childNodes:
xml.write(node.toxml())
new_xml_file = ContentFile(xml.getvalue())
new_xml_file.content_type = 'text/xml'
xml.close()
attachments = []
for attach in de_node.getElementsByTagName('mediaFile'):
filename_node = attach.getElementsByTagName('filename')
filename = filename_node[0].childNodes[0].nodeValue
if filename in files:
file_obj = default_storage.open(
os.path.join(instance_dir_path, filename))
mimetype, encoding = mimetypes.guess_type(file_obj.name)
media_obj = django_file(file_obj, 'media_files[]', mimetype)
attachments.append(media_obj)
create_instance(self.user.username, new_xml_file, attachments)
def get_pdf(self, issuer=None, language=None):
"""Return the cached PDF or generate and cache it."""
from core.models import VosaeFile
try:
pdf_language = language or self.tenant.report_settings.language
except:
pdf_language = 'en'
if not self.current_revision.pdf.get(pdf_language, None):
with respect_language(pdf_language):
buf = self.gen_pdf()
pdf = ContentFile(buf.getvalue(), self.filename)
pdf.content_type = "application/pdf"
self.current_revision.pdf[pdf_language] = VosaeFile(
tenant=self.tenant,
uploaded_file=pdf,
issuer=issuer
)
self.current_revision.pdf[pdf_language].save()
self.update(set__current_revision__pdf=self.current_revision.pdf)
return self.current_revision.pdf[pdf_language]
def get_pdf(self, issuer=None, language=None):
"""Return the cached PDF or generate and cache it."""
from core.models import VosaeFile
try:
pdf_language = language or self.tenant.report_settings.language
except:
pdf_language = 'en'
if not self.current_revision.pdf.get(pdf_language, None):
with respect_language(pdf_language):
buf = self.gen_pdf()
pdf = ContentFile(buf.getvalue(), self.filename)
pdf.content_type = "application/pdf"
self.current_revision.pdf[pdf_language] = VosaeFile(
tenant=self.tenant,
uploaded_file=pdf,
issuer=issuer
)
self.current_revision.pdf[pdf_language].save()
self.update(set__current_revision__pdf=self.current_revision.pdf)
return self.current_revision.pdf[pdf_language]
def export_documents(export):
from contacts.imex import get_exportable_documents as contacts_documents
from invoicing.imex import get_exportable_documents as invoicing_documents
with respect_language(export.language):
archive_name = _('Vosae export.zip')
zipped = ContentFile('', archive_name)
f = zipfile.ZipFile(zipped, mode='w', compression=zipfile.ZIP_DEFLATED)
for documents, path_func, doc_func in contacts_documents(export):
for document in documents:
f.writestr(path_func(document), doc_func(document))
for documents, path_func, doc_func in invoicing_documents(export):
for document in documents:
f.writestr(path_func(document), doc_func(document))
f.close()
zipped.content_type = "application/zip"
export.zipfile = VosaeFile(
tenant=export.tenant,
uploaded_file=zipped,
issuer=export.issuer
)
export.zipfile.save()
export.update(set__zipfile=export.zipfile)
context = {
'tenant': export.tenant,
'file': export.zipfile,
'site': {'name': settings.SITE_NAME, 'url': settings.SITE_URL}
}
# Email to issuer
plaintext_context = Context(autoescape=False) # HTML escaping not appropriate in plaintext
subject = subject = _("Your Vosae export is available")
text_body = render_to_string('data_liberation/emails/export_finished.txt', context, plaintext_context)
html_body = render_to_string("data_liberation/emails/export_finished.html", context)
message = EmailMessage(subject=subject, from_email=settings.DEFAULT_FROM_EMAIL,
to=[export.issuer.email], body=text_body)
message.attach_alternative(html_body, "text/html")
message.send()
def create_report_photo(request):
'''This method is used to create citizens reports. Validation included.'''
#Test the submit content size (max 2MB)
if (int(request.META.get('CONTENT_LENGTH')) > 15000000):
return HttpResponseBadRequest(json.dumps({
"error_key": "ERROR_REPORT_FILE_EXCEED_SIZE",
"request": request.POST
}),
content_type='application/json')
data_report_id = request.POST.get('report_id')
# Check if the photo is base64 encoded
if request.POST.get('base64', False):
image_data_base64 = request.POST.get('report_file')
# Extract content and metadata from base64 info
metadata, file_base64 = image_data_base64.split(';base64,')
metadata = metadata.replace("data:", "")
# Extract name and type from metadata
data_file_type, data_file_name = metadata.split(',')
# Set (meta)data to file object
data_file_content = ContentFile(file_base64.decode('base64'),
name=data_file_name)
data_file_content.content_type = data_file_type
else:
data_file_content = request.FILES.get('report_file')
#Verify that everything has been posted to create a citizen report.
if (data_report_id == None):
return HttpResponseBadRequest(json.dumps({
"error_key": "ERROR_REPORT_FILE_MISSING_DATA_REPORT_ID",
"request": request.POST
}),
content_type='application/json')
if (data_file_content == None):
return HttpResponseBadRequest(json.dumps({
"error_key": "ERROR_REPORT_FILE_MISSING_DATA_REPORT_FILE",
"request": request.POST
}),
content_type='application/json')
try:
#Retrieve the report
reference_report = Report.objects.get(id=data_report_id)
except Exception:
return HttpResponseBadRequest(json.dumps({
"error_key": "ERROR_REPORT_FILE_NOT_FOUND",
"request": request.POST
}),
content_type='application/json')
report_file = ReportFile()
try:
report_file.title = "Mobile Upload"
report_file.file = data_file_content
report_file.report = reference_report
report_file.file_creation_date = datetime.now()
# Set same creator than report creator
report_file.created_by = reference_report.citizen or reference_report.created_by
#Save given data
report_file.is_new_report = True
report_file.save()
except Exception:
return HttpResponseBadRequest(json.dumps({
"error_key": "ERROR_REPORT_FILE_PROBLEM_DATA",
"request": request.POST
}),
content_type='application/json')
#Return the report ID
return JsonHttpResponse({'report_photo': report_file.id})
def get_file_for_page(self, template, context):
context['staticgenerator'] = True
html = render_to_string(template, context)
f = ContentFile(html, name="index.html")
f.content_type = "text/html;charset=utf-8"
return f
def create_fake_upload():
fake_upload = ContentFile(fake.binary(length=fake.random_int()))
fake_upload.name = fake.file_name()
fake_upload.content_type = fake.mime_type()
return fake_upload
