def test_05_checksum_conversion(self):
"""
Test Conversion Cache return expected value with checksum
"""
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document = self.portal.portal_contributions.newContent(file=file_)
self.tic()
document_url = document.getRelativeUrl()
document = self.portal.restrictedTraverse(document_url)
kw = {'format': 'html'}
#Generate one conversion
document.convert(**kw)
cache_id = document._getCacheKey(**kw)
cache_factory = document._getCacheFactory()
data_dict = cache_factory.get(cache_id)
#get data from cache
self.assertTrue(data_dict['content_md5'])
self.assertTrue(data_dict['conversion_md5'])
self.assertTrue(data_dict['mime'])
self.assertTrue(data_dict['data'])
self.assertTrue(data_dict['date'])
self.assertTrue(data_dict['size'])
#Change md5 manualy
data_dict['content_md5'] = 'Anything which is not md5'
cache_factory.set(cache_id, data_dict)
self.commit()
self.assertRaises(KeyError, document.getConversion, format='html')
def test_Base_isConvertible(self):
"""
Test pre converion only happens on proper documents.
"""
image = self.portal.image_module.newContent(portal_type='Image',
reference='Embedded-XXX',
version='001',
language='en')
# draft image is not convertible
upload_file = makeFileUpload('cmyk_sample.jpg')
image.edit(file=upload_file)
self.tic()
self.assertEqual(False, image.Base_isConvertible())
# published image with data is convertible
image.publish()
self.tic()
self.assertEqual(True, image.Base_isConvertible())
image = self.portal.image_module.newContent(portal_type='Image',
reference='Embedded-YYY',
version='001',
language='en')
image.publish()
self.tic()
# published empty image is not convertible
self.assertEqual(False, image.Base_isConvertible())
def test_preConvertedReferencedImageInWebPageContent(self):
# create an image
upload_file = makeFileUpload('cmyk_sample.jpg')
image = self.portal.image_module.newContent(portal_type='Image',
reference='Embedded-XXX',
version='001',
language='en')
image.edit(file=upload_file)
image.publish()
self.tic()
# after a web page creation, the referenced images are found and should be pre converted
# with the parameters given in their src URL
web_page = self.portal.web_page_module.newContent(portal_type="Web Page")
web_page.setTextContent('''<b> test </b>
<img src="Embedded-XXX?format=png&display=large&quality=64"/>
<img src="Embedded-XXX?format=jpeg&display=large&quality=64"/>''')
web_page.publish()
self.tic()
# check that referenced in Web Page's content image(s) is well converted
self.assertTrue(image.hasConversion(**{'format':'jpeg', 'display':'large', 'quality':64}))
self.assertTrue(image.hasConversion(**{'format':'png', 'display':'large', 'quality':64}))
self.assertSameSet(['Embedded-XXX?format=png&display=large&quality=64', \
'Embedded-XXX?format=jpeg&display=large&quality=64'],
web_page.Base_extractImageUrlList())
def test_erp5_merge_pdf(self):
document = self.portal.portal_contributions.newContent(
file=makeFileUpload('REF-en-001.pdf'))
merged_pdf_data = self.portal.ERP5Site_mergePDFList(
[document.getData(), document.getData()])
merged_document = self.portal.document_module.newContent(
portal_type='PDF', data=merged_pdf_data)
self.assertEqual('2', merged_document.getContentInformation()['Pages'])
def test_06_check_md5_is_updated(self):
"""
Check that md5 checksum is well updated when upload a file
"""
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document = self.portal.portal_contributions.newContent(file=file_)
self.tic()
document_url = document.getRelativeUrl()
document = self.portal.restrictedTraverse(document_url)
md5sum = document.getContentMd5()
self.assertTrue(md5sum)
filename2 = 'TEST-en-002.odt'
file2 = makeFileUpload(filename2)
document.edit(file=file2)
self.assertNotEqual(md5sum, document.getContentMd5())
self.tic()
def test_erp5_merge_pdf_start_on_recto(self):
document = self.portal.portal_contributions.newContent(
file=makeFileUpload('REF-en-001.pdf'))
merged_pdf_data = self.portal.ERP5Site_mergePDFList(
[document.getData(), document.getData()], start_on_recto=True)
merged_document = self.portal.document_module.newContent(
portal_type='PDF', data=merged_pdf_data)
# there are four pages, because blank pages has been added so that the
# second time, our document starts on a recto page.
self.assertEqual('4', merged_document.getContentInformation()['Pages'])
def test_03_CacheConversionOfTempObjectIsNotMixed(self):
filename1 = 'TEST-en-002.doc'
filename2 = 'TEST-en-002.odt'
file1 = makeFileUpload(filename1)
file2 = makeFileUpload(filename2)
document1 = self.portal.portal_contributions.newContent(file=file1,
temp_object=1)
document1.uploadFile()
document1.processFile()
document1.convertToBaseFormat()
document2 = self.portal.portal_contributions.newContent(file=file2,
temp_object=1)
document2.uploadFile()
document2.processFile()
document2.convertToBaseFormat()
format_ = 'pdf'
document1.convert(format=format_)
document2.convert(format=format_)
self.assertNotEqual(document1.getConversion(format=format_),
document2.getConversion(format=format_))
self.tic()
def test_01_no_retry_for_no_network_issue(self):
system_pref = self.getDefaultSystemPreference()
system_pref.setPreferredDocumentConversionServerRetry(self.retry_count)
self.tic()
filename = 'monochrome_sample.tiff'
file_ = makeFileUpload(filename)
document = self.portal.document_module.newContent(portal_type='Text')
document.edit(file=file_)
message = document.Document_tryToConvertToBaseFormat()
self.assertEqual(
message.count('Error converting document to base format'), 1)
def test_03_retry_for_socket_issue(self):
system_pref = self.getDefaultSystemPreference()
server_list = system_pref.getPreferredDocumentConversionServerUrlList()
system_pref.setPreferredDocumentConversionServerRetry(self.retry_count)
system_pref.setPreferredOoodocServerTimeout(1)
self.tic()
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document = self.portal.portal_contributions.newContent(file=file_)
message = document.Document_tryToConvertToBaseFormat()
if 'Socket Error: SSLError' in message:
self.assertEqual(message.count('Socket Error: SSLError'),
(self.retry_count + 1) * len(server_list))
def test_showwarning_issue(self):
"""
We should not let PdfFileReader overwrite warnings.showwarning method because we already do it in ERP5
https://github.com/mstamy2/PyPDF2/blob/18a2627adac13124d4122c8b92aaa863ccfb8c29/PyPDF2/pdf.py#L1129
"""
self.assertEquals(Utils._showwarning, warnings.showwarning)
document = self.portal.portal_contributions.newContent(
file=makeFileUpload('REF-en-001.pdf'))
merged_pdf_data = self.portal.ERP5Site_mergePDFList(
[document.getData(), document.getData()])
self.portal.document_module.newContent(portal_type='PDF',
data=merged_pdf_data)
self.tic()
self.assertEquals(Utils._showwarning, warnings.showwarning)
def test_Zuite_uploadScreenShot(self):
"""
Test Screeshot upload script used by Zelenium to
update screenshots of the documents.
"""
image_upload = makeFileUpload('TEST-en-002.png')
self.assertNotEquals(None, image_upload)
# Create a web page, and check if the content is not overwriten
web_page_reference = "WEB-PAGE-REFERENCE"
web_page = self.portal.web_page_module.newContent(
reference=web_page_reference,
language="en", version="001")
web_page.publishAlive()
self.tic()
image_reference = "IMAGE-REFERENCE-%s" % str(time())
image_page = self.portal.image_module.newContent(
reference=image_reference,
language="en", version="001")
image_page.publishAlive()
self.tic()
image_page_2 = self.portal.image_module.newContent(
reference=image_reference,
language="en", version="002")
image_page_2.publishAlive()
self.tic()
self.portal.REQUEST.form['data_uri'] = image_upload
fake_image_reference = "DO-NOT-EXISTANT-IMAGE"
self.assertNotEquals(None,
self.portal.Zuite_uploadScreenshot(image_upload, fake_image_reference))
self.assertNotEquals(None,
self.portal.Zuite_uploadScreenshot(image_upload, web_page_reference))
self.assertEqual(None,
self.portal.Zuite_uploadScreenshot(image_upload, image_reference))
self.tic()
# The right image were updated.
image_upload.seek(0)
self.assertEqual(image_page_2.getData(), image_upload.read().decode("base64"))
self.assertEqual(image_page_2.getFilename(), image_reference + '.png')
self.assertEqual(image_page.getData(), '')
def test_02_retry_for_network_issue(self):
system_pref = self.getDefaultSystemPreference()
saved_server_list = system_pref.getPreferredDocumentConversionServerUrlList(
)
system_pref.setPreferredDocumentConversionServerRetry(self.retry_count)
system_pref.setPreferredDocumentConversionServerUrlList(
['https://broken.url'])
self.tic()
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document = self.portal.portal_contributions.newContent(file=file_)
message = document.Document_tryToConvertToBaseFormat()
self.assertEqual(message.count('broken.url: Connection refused'),
self.retry_count + 1)
system_pref.setPreferredDocumentConversionServerUrlList(
saved_server_list)
self.commit()
def test_01_PersistentCacheConversion(self):
"""
Test Conversion Cache mechanism
"""
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document = self.portal.portal_contributions.newContent(file=file_)
self.tic()
document_url = document.getRelativeUrl()
document = self.portal.restrictedTraverse(document_url)
format_list = [
format_ for format_ in document.getTargetFormatList()
if format_ not in self.failed_format_list
]
if not format_list:
self.fail('Target format list is empty')
#Test Conversion Cache
for format_ in format_list:
document.convert(format=format_)
self.commit()
self.assertTrue(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertEqual(DateTime().Date(),
document.getConversionDate(format=format_).Date())
self.assertTrue(document.getConversionMd5(format=format_))
self.assertTrue(document.getConversionSize(format=format_))
document.edit(title='Foo')
self.commit()
#Test Cache is cleared
for format_ in format_list:
self.assertFalse(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertRaises(KeyError,
document.getConversionSize,
format=format_)
document.edit(title='Bar')
self.tic()
#Test Conversion Cache after editing
for format_ in format_list:
document.convert(format=format_)
self.commit()
self.assertTrue(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertTrue(document.getConversionSize(format=format_))
def test_AttachmentIngestion(self):
"""
Test the attachment of a CSV file, from both newDiscussionPost and newDiscussionThread
use cases.
CSV wasn't chosen randomly, as it may be subjected to a portal type migration through
discover metadata, which used to cause a bug.
"""
discussion_thread_id_set = set(
self.portal.discussion_thread_module.objectIds())
web_site_value = self.portal.web_site_module.newContent(
portal_type='Web Site')
web_section_value = web_site_value.newContent(
portal_type='Web Section')
file_ = makeFileUpload('simple.csv')
web_section_value.WebSection_createNewDiscussionThread("Thread Title",
"Post Content",
file=file_)
self.tic()
thread_value, = [
x for x in self.portal.discussion_thread_module.objectValues()
if x.getId() not in discussion_thread_id_set
]
post_value, = thread_value.objectValues(portal_type='Discussion Post')
tested_post_value_set = {
post_value,
}
attachment_list = post_value.DiscussionPost_getAttachmentList()
self.assertEqual(1, len(attachment_list))
thread_value.DiscussionThread_createNewDiscussionPost(
title="Post Title",
text_content="Post Content",
file=file_,
)
self.tic()
post_value, = [
x for x in thread_value.objectValues()
if x not in tested_post_value_set
]
attachment_list = post_value.DiscussionPost_getAttachmentList()
self.assertEqual(1, len(attachment_list))
def test_02_VolatileCacheConversionOfTempObject(self):
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document = self.portal.portal_contributions.newContent(file=file_,
temp_object=1)
document.uploadFile()
document.processFile()
document.convertToBaseFormat()
format_list = [
format_ for format_ in document.getTargetFormatList()
if format_ not in self.failed_format_list
]
if not format_list:
self.fail('Target format list is empty')
#Test Conversion Cache
for format_ in format_list:
document.convert(format=format_)
self.commit()
self.assertTrue(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertEqual(DateTime().Date(),
document.getConversionDate(format=format_).Date())
self.assertTrue(document.getConversionMd5(format=format_))
self.assertTrue(document.getConversionSize(format=format_))
document.edit(title='Foo')
self.commit()
#Test Cache is cleared
for format_ in format_list:
self.assertFalse(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertRaises(KeyError,
document.getConversionSize,
format=format_)
document.edit(title='Bar')
self.tic()
#Test Conversion Cache after editing
for format_ in format_list:
document.convert(format=format_)
self.commit()
self.assertTrue(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertTrue(document.getConversionSize(format=format_))
def test_04_PersistentCacheConversionWithFlare(self):
default_pref = self.portal.portal_preferences.default_site_preference
default_pref.setPreferredConversionCacheFactory('dms_cache_factory')
#old preferred value is still cached
self.portal.portal_caches.clearAllCache()
self.tic()
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document = self.portal.portal_contributions.newContent(file=file_)
self.tic()
document_url = document.getRelativeUrl()
document = self.portal.restrictedTraverse(document_url)
format_list = [format_ for format_ in document.getTargetFormatList()\
if format_ not in self.failed_format_list]
if not format_list:
self.fail('Target format list is empty')
#Test Conversion Cache
for format_ in format_list:
document.convert(format=format_)
self.assertTrue(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertTrue(document.getConversionSize(format=format_))
document.edit(title='Foo')
self.commit()
#Test Cache is cleared
for format_ in format_list:
self.assertFalse(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertRaises(KeyError,
document.getConversionSize,
format=format_)
document.edit(title='Bar')
self.tic()
#Test Conversion Cache after editing
for format_ in format_list:
document.convert(format=format_)
self.assertTrue(document.hasConversion(format=format_),
'Cache Storage failed for %s' % (format_))
self.assertTrue(document.getConversionSize(format=format_))
def test_image_conversion(self):
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document = self.portal.portal_contributions.newContent(file=file_)
self.tic()
format_ = 'png'
self.assertFalse(document.hasConversion(format=format_))
document.convert(format_)
self.assertTrue(document.hasConversion(format=format_))
self.assertFalse(
document.hasConversion(format=format_, display='large'))
document.convert(format_, display='large')
self.assertTrue(document.hasConversion(format=format_,
display='large'))
self.assertFalse(
document.hasConversion(format=format_, display='large',
quality=40))
document.convert(format_, display='large', quality=40)
self.assertTrue(
document.hasConversion(format=format_, display='large',
quality=40))
if magic is not None:
mime_detector = magic.Magic(mime=True)
self.assertEqual(
mime_detector.from_buffer(
document.getConversion(format=format_)[1]), 'image/png')
self.assertEqual(
mime_detector.from_buffer(
document.getConversion(format=format_,
display='large')[1]), 'image/png')
self.assertEqual(
mime_detector.from_buffer(
document.getConversion(format=format_,
display='large',
quality=40)[1]), 'image/png')
def test_08_check_conversion_cache_with_portal_document_type_list(self):
"""Check cache conversion for all Portal Document Types
"""
portal_type_list = list(self.portal.getPortalDocumentTypeList())
if 'File' in portal_type_list:
#File conversion is not implemented
portal_type_list.remove('File')
if 'Web Illustration' in portal_type_list:
#Web Illustration conversion is not implemented
portal_type_list.remove('Web Illustration')
if 'Web Table' in portal_type_list:
#Web Table conversion is not implemented
portal_type_list.remove('Web Table')
data_mapping = {
'Drawing': 'TEST-en-002.sxd',
'Text': 'TEST-en-002.doc',
'Spreadsheet': 'TEST-en-002.sxc',
'Presentation': 'TEST-en-002.sxi',
'Web Page': 'TEST-en-002.html',
'Image': 'TEST-en-002.gif',
'File': 'TEST-en-002.rtf',
'PDF': 'TEST-en-002.pdf'
}
#Check that all portal_types are handled by test
self.assertEqual(
len(portal_type_list),
len([pt for pt in portal_type_list if pt in data_mapping]))
for portal_type in portal_type_list:
module = self.portal.getDefaultModule(portal_type=portal_type)
upload_file = makeFileUpload(data_mapping[portal_type])
document = module.newContent(portal_type=portal_type)
document.edit(file=upload_file)
self.tic()
document.convert(format='txt')
document.convert(format='html')
self.assertTrue(document.getConversion(format='txt'))
self.assertTrue(document.getConversion(format='html'))
def test_07_check_cache_key_is_escaped(self):
"""
Check that key (based on path of document) support unauthorised chars
"""
default_pref = self.portal.portal_preferences.default_site_preference
default_pref.setPreferredConversionCacheFactory('dms_cache_factory')
#old preferred value is still cached
self.portal.portal_caches.clearAllCache()
self.tic()
filename = 'TEST-en-002.doc'
file_ = makeFileUpload(filename)
document_id = 'an id with spaces'
portal_type = 'Text'
module = self.portal.getDefaultModule(portal_type)
document = module.newContent(id=document_id,
file=file_,
portal_type=portal_type)
self.tic()
document_url = document.getRelativeUrl()
document = self.portal.restrictedTraverse(document_url)
self.assertEqual(document.getId(), document_id)
document.convert(format='txt')
self.assertTrue(document.getConversion(format='txt'))
def test_03_createDiscussionThread(self):
"""
Create a disucssion thread
"""
portal = self.portal
discussion_thread_id_set = set(
portal.discussion_thread_module.objectIds())
# create web sections & set predicates
group1 = portal.portal_categories.group.newContent(
portal_type='Category', title='Group 1')
web_site = portal.web_site_module.newContent(portal_type='Web Site')
web_section1 = web_site.newContent(portal_type='Web Section')
web_section1.setMultimembershipCriterionBaseCategoryList(['group'])
web_section1.setMembershipCriterionCategoryList(
[group1.getRelativeUrl()])
self.tic()
web_section1.WebSection_createNewDiscussionThread(
'test1-new', 'test1 body')
discussion_thread, = [x for x in self.portal.discussion_thread_module.objectValues() \
if x.getId() not in discussion_thread_id_set]
discussion_thread_id_set.add(discussion_thread.getId())
self.assertTrue(
discussion_thread.getReference().startswith("test1-new-"))
# not indexed yet
self.assertSameSet([],
web_section1.WebSection_getDiscussionThreadList())
# not indexed but its relative url is passed through REQUEST
self.app.REQUEST.set('thread_relative_url',
discussion_thread.getRelativeUrl())
self.assertSameSet([discussion_thread],
web_section1.WebSection_getDiscussionThreadList())
self.tic()
# indexed already
self.assertSameSet([discussion_thread],
web_section1.WebSection_getDiscussionThreadList())
discussion_post = discussion_thread.contentValues(
filter={'portal_type': 'Discussion Post'})[0]
attachment_list = discussion_post.DiscussionPost_getAttachmentList()
self.assertEqual(discussion_thread.getValidationState(), 'published')
self.assertEqual(0, len(attachment_list))
# check attachment creation
file_ = makeFileUpload('TEST-en-002.doc')
web_section1.WebSection_createNewDiscussionThread(
'test1-new-with-attachment', 'test1 body', file=file_)
discussion_thread, = [x for x in self.portal.discussion_thread_module.objectValues() \
if x.getId() not in discussion_thread_id_set]
discussion_thread_id_set.add(discussion_thread.getId())
self.assertTrue(discussion_thread.getReference().startswith(
"test1-new-with-attachment-"))
self.tic()
discussion_post = discussion_thread.contentValues(
filter={'portal_type': 'Discussion Post'})[0]
attachment_list = discussion_post.DiscussionPost_getAttachmentList()
self.assertEqual(discussion_thread.getValidationState(), 'published')
self.assertEqual(1, len(attachment_list))