def test_storage_removes_obfuscation(self):
gsettings = GlobalSettings(self.portal)
_dir = mkdtemp()
gsettings.storage_location = _dir
gsettings.storage_type = 'File'
gsettings.storage_obfuscate = True
fi = self.createFile('test.pdf')
uid = fi.UID()
fi.reindexObject()
notify(ObjectModifiedEvent(fi))
settings = Settings(fi)
self.assertTrue(
exists(join(_dir, uid[0], uid[1], uid,
settings.obfuscate_secret)))
self.assertTrue(
len(listdir(join(_dir, uid[0], uid[1], uid,
settings.obfuscate_secret))) > 3)
# publish now
workflowTool = getToolByName(fi, 'portal_workflow')
workflowTool.doActionFor(fi, 'publish')
self.assertTrue(
not exists(join(_dir, uid[0], uid[1], uid,
settings.obfuscate_secret)))
def test_indexation_switch_mode(self):
'''
Test that switching the indexation from enabled to disabled
and the other way round keep the state consistent.
'''
fi = self.createFile('test.pdf')
# indexation is enabled by default
# make sure conversion was successfull
self.failUnless(self._isSuccessfullyConverted(fi))
annotations = IAnnotations(fi)['collective.documentviewer']
# something is catalogued
self.failUnless(annotations['catalog'] is not None)
# now disable indexation and convert again
gsettings = GlobalSettings(self.portal)
gsettings.enable_indexation = False
# make it convertable again by adapting last_updated and filehash
annotations['last_updated'] = DateTime('1901/01/01').ISO8601()
annotations['filehash'] = 'dummymd5'
notify(ObjectInitializedEvent(fi))
# make sure conversion was successfull
self.failUnless(self._isSuccessfullyConverted(fi))
# nothing indexed anymore
self.failIf(annotations['catalog'] is not None)
def __call__(self):
""" Redirects to the url for the rendered PDF.
We need to redirect, because the PDF can be stored on FS, instead
of ZODB.
"""
site = getPortal(self.context)
settings = Settings(self.context)
global_settings = GlobalSettings(site)
portal_url = getMultiAdapter((self.context, self.request),
name="plone_portal_state").portal_url()
resource_url = global_settings.override_base_resource_url
rel_url = storage.getResourceRelURL(gsettings=global_settings,
settings=settings)
if resource_url:
dvpdffiles = '%s/%s' % (resource_url.rstrip('/'), rel_url)
else:
dvpdffiles = '%s/%s' % (portal_url, rel_url)
url = '%s/pdf/dump.pdf' % dvpdffiles
self.request.response.redirect(url)
def get_previews(self, obj):
if IImage.providedBy(obj):
return {
'class': '',
'url': '{}/@@images/image'.format(obj.absolute_url()),
'page_count': 1,
'has_preview': True,
}
portal = api.portal.get()
gsettings = GlobalSettings(portal)
preview_details = {'class': 'not-generated', 'has_preview': False}
if not allowedDocumentType(obj, gsettings.auto_layout_file_types):
preview_details['class'] = 'not-possible'
elif previews.has_previews(obj):
preview_urls = previews.get_preview_urls(obj)
if preview_urls:
preview_details = {
'class': '',
'url': preview_urls[0],
'page_count': len(preview_urls),
'has_preview': True,
}
return preview_details
def convert_all(context):
catalog = getToolByName(context, 'portal_catalog')
portal = getSite()
gsettings = GlobalSettings(portal)
for brain in catalog(object_provides=IFileContent.__identifier__):
file = brain.getObject()
if not allowedDocumentType(file, gsettings.auto_layout_file_types):
continue
# let's not switch to the document viewer view
# until the document is converted. The conversion
# process will check if the layout is set correctly.
if file.getLayout() != 'documentviewer':
settings = Settings(file)
settings.last_updated = DateTime('1999/01/01').ISO8601()
queueJob(file)
else:
settings = Settings(file)
# also convert if there was an error.
if settings.successfully_converted == False:
settings.last_updated = DateTime('1999/01/01').ISO8601()
settings.filehash = ''
queueJob(file)
def test_not_auto_assigns_view(self):
fi = self.createFile('test.pdf')
gsettings = GlobalSettings(self.portal)
gsettings.auto_select_layout = False
notify(ObjectInitializedEvent(fi))
self.assertTrue(fi.getLayout() != 'documentviewer')
def test_not_auto_assigns_view(self):
gsettings = GlobalSettings(self.portal)
gsettings.auto_select_layout = False
fi = self.createFile('test.pdf')
self.assertTrue(fi.getLayout() != 'documentviewer')
def change_settings(context):
''' Update the document converter settings
'''
global_settings = GlobalSettings(api.portal.get())
global_settings.thumb_size = 460
def _get_dv_data(obj):
""" Access the collective.documentviewer settings of an object and
return metadata that can be used to retrieve or access a preview
image.
:param obj: The Plone content object that has a preview
:type obj: A Plone content object
:return: Metadata consisting of canonical_url, number of pages and
resource urls to preview images and thumbnails.
:rtype: mapping
"""
site = getSite()
global_settings = GlobalSettings(site)
settings = Settings(obj)
portal_url = site.portal_url()
try:
canonical_url = obj.absolute_url()
except AttributeError:
canonical_url = '' # XXX construct a url to an attachment
if not hasattr(obj, 'UID') or not settings.successfully_converted:
# Can't have previews on objects without a UID. Return a
# minimal datastructure
return {
'canonical_url': canonical_url + '/view',
'pages': settings.num_pages,
'resources': {
'page': {
'image': fallback_image_url(obj),
'text': ''
},
'pdf': canonical_url,
'thumbnail': fallback_image_url(obj),
}
}
resource_url = global_settings.override_base_resource_url
rel_url = storage.getResourceRelURL(gsettings=global_settings,
settings=settings)
if resource_url:
dvpdffiles = '%s/%s' % (resource_url.rstrip('/'),
rel_url)
else:
dvpdffiles = '%s/%s' % (portal_url, rel_url)
dump_path = DUMP_FILENAME.rsplit('.', 1)[0]
image_format = settings.pdf_image_format or \
global_settings.pdf_image_format
return {
'canonical_url': canonical_url + '/view',
'pages': settings.num_pages,
'resources': {
'page': {
'image': '%s/{size}/%s_{page}.%s' % (
dvpdffiles, dump_path,
image_format),
'text': '%s/%s/%s_{page}.txt' % (
dvpdffiles, TEXT_REL_PATHNAME, dump_path)
},
'pdf': canonical_url,
'thumbnail': '%s/small/%s_1.%s' % (
dvpdffiles, dump_path,
image_format),
}
}
def default_show_search():
# take the value from the global settings
from collective.documentviewer.settings import GlobalSettings
gsettings = GlobalSettings(getSite())
return gsettings.show_search
def default_enable_indexation():
# take the value from the global settings
from collective.documentviewer.settings import GlobalSettings
gsettings = GlobalSettings(getSite())
return gsettings.enable_indexation
def __call__(self):
portal = api.portal.get()
gsettings = GlobalSettings(portal)
catalog = portal.portal_catalog
logger.info(
'Parsing UIDs in the portal_catalog to find deleted objects.')
items = catalog.Indexes['UID'].referencedObjects()
cpt = 0
logger.info('Parsing ' + str(len(items)) + ' objects.')
for item in items:
url = catalog.getpath(item)
try:
obj = catalog.unrestrictedTraverse(url)
except AttributeError:
logger.info('Object doesn\'t exist anymore : ' + url)
catalog.uncatalog_object(url)
cpt += 1
logger.info(str(cpt) + ' objects parsed.')
logger.info(
'Parsing MeetingFiles and CourrierFiles to see if all is correctly converted.'
)
annexUIDs = set()
brains = catalog.searchResults(meta_type='MeetingFile',
sort_on='Date',
sort_order='ascending')
for brain in brains:
try:
annex = brain.getObject()
annexUID = annex.UID()
path = join(gsettings.storage_location, annexUID[0],
annexUID[1], annexUID)
if not os.path.exists(path):
logger.info(annex.absolute_url() +
' : Annex not converted !')
annexUIDs.add(annexUID)
except AttributeError:
url = brain.getPath()
logger.info('Object doesn\'t exist anymore : ' + url)
catalog.uncatalog_object(url)
logger.info(str(len(annexUIDs)) + ' annexes in the database')
mailUIDs = set()
brains = catalog.searchResults(meta_type='CourrierFile',
sort_on='Date',
sort_order='descending')
for brain in brains:
mail = brain.getObject()
mailUID = mail.UID()
path = join(gsettings.storage_location, mailUID[0], mailUID[1],
mailUID)
if not os.path.exists(path):
logger.info(mail.absolute_url() + ' : Mail not converted !')
mailUIDs.add(mailUID)
logger.info(str(len(mailUIDs)) + ' mails in the database')
logger.info(
'Parsing all UIDs to see if all converted files are linked to objects.'
)
cpt = 0
convertedUIDs = annexUIDs.union(mailUIDs)
hexachars = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a',
'b', 'c', 'd', 'e', 'f')
for firstchar in hexachars:
for secondchar in hexachars:
path = join(gsettings.storage_location, firstchar, secondchar)
if os.path.exists(path):
for uid in os.listdir(path):
uidpath = join(gsettings.storage_location, firstchar,
secondchar, uid)
if not os.path.isdir(uidpath):
logger.info(
'There\'s something strange in the directories hierarchy : '
+ uidpath)
else:
if not uid in convertedUIDs:
logger.info(
'Removing unused converted annex : ' +
uidpath)
shutil.rmtree(uidpath)
cpt += 1
logger.info(
str(cpt) + ' directories removed from the conversion results')
def __call__(self):
self._update()
add_resource_on_request(self.request, 'documentviewer')
self.site = api.portal.get()
self.settings = Settings(self.context)
self.global_settings = GlobalSettings(self.site)
self.portal_url = getMultiAdapter(
(self.context, self.request),
name="plone_portal_state").portal_url()
self.dvstatic = "%s/++resource++dv.resources" % (self.portal_url)
resource_url = self.global_settings.override_base_resource_url
rel_url = storage.getResourceRelURL(gsettings=self.global_settings,
settings=self.settings)
if resource_url:
self.dvpdffiles = '%s/%s' % (resource_url.rstrip('/'), rel_url)
else:
self.dvpdffiles = '%s/%s' % (self.portal_url, rel_url)
utils = getToolByName(self.context, 'plone_utils')
msg = None
self.enabled = True
if allowedDocumentType(self.context,
self.global_settings.auto_layout_file_types):
if not self.installed:
msg = _(
"Since you do not have docsplit installed on this "
"system, we can not render the pages of this document.")
if self.settings.converting is not None and \
self.settings.converting:
if self.settings.successfully_converted:
# there is a version that is already converted, show it.
self.enabled = True
msg = _("A new conversion to the Document Viewer "
"is currently being generated for this document.")
else:
msg = _("The document is currently being converted to the "
"Document Viewer view.")
self.enabled = False
elif self.settings.successfully_converted is not None and \
not self.settings.successfully_converted:
msg = _("There was an error trying to convert the document. "
"Maybe the document is encrypted, corrupt or "
"malformed? Check log for details.")
self.enabled = False
elif self.settings.successfully_converted is None:
# must have just switched to this view
msg = _("This document is not yet converted to document "
"viewer. Please click the `Document Viewer Convert` "
"button in the actions menu to convert.")
self.enabled = False
else:
self.enabled = False
msg = _("The file is not a supported document type. "
"Your type may be supported. Check out the document "
"viewer configuration settings.")
mtool = getToolByName(self.context, 'portal_membership')
self.can_modify = mtool.checkPermission('Modify portal content',
self.context)
if msg and self.can_modify:
utils.addPortalMessage(_(msg))
return self.index()
def convertFileToPdf(self, repositoryitem, fileobj):
"""
Convert a single file to document viewer using collective.documentviewer
"""
# Check that docviewer is installed before processing
if docViewerInstalled():
from collective.documentviewer.convert import Converter
from collective.documentviewer.settings import GlobalSettings
from zope.app.component.hooks import getSite
from bungenicms.repository.browser.interfaces import IEnhancedDocumentViewerSchema
context = repositoryitem
filename = fileobj["filename"]
portal_url = getToolByName(context, "portal_url")()
isFileConverted = False
# Where to put in the newly created objects
# Get the settings from collective.documentviewer
gsettings = GlobalSettings(getSite())
gsettings.use_interface = IEnhancedDocumentViewerSchema
storage_folder = gsettings.folder_location or CONVERTED_FILES_FOLDER # if empty
plone_root = context.portal_url.getPortalObject()
container = None
if plone_root.hasObject(storage_folder):
container = plone_root[storage_folder]
print "files-folder exists. No need to create a new one."
else:
plone_root.invokeFactory("Folder", id=storage_folder, title="Files Folder")
container = plone_root[storage_folder]
print CONVERTED_FILES_FOLDER + " does NOT exist. Created a new one."
# Make sure the folder is public/published
try:
folder_review_state = container.portal_workflow.getInfoFor(container, "review_state")
if not folder_review_state == "published":
container.portal_workflow.doActionFor(container, "publish", comment="published")
except:
print "Could not publish: " + str(container.getId) + " already published?"
# Confirm whether the file has been converted using object UID
uid = None
for id, item in container.objectItems():
if context.UID() == "FILE".join( item.UID().split( "FILE" )[0:1] ):
if filename.translate(None, " ?.!/\;:-{}[]()|~^`") == item.UID().split("FNIX",1)[1]:
print "A file with the same name already exists. No need to re-convert."
isFileConverted = True
uid = item.UID()
break
if not isFileConverted:
# Grant user temp managerial permssions to allow doc conversion.
sm = getSecurityManager()
if "Manager" not in sm.getUser().getRoles():
tmp_user = BaseUnrestrictedUser(
sm.getUser().getId(),"", ["Manager"],"")
newSecurityManager(None, tmp_user)
# Set the file object attributes using the format below:
# [parent_UID]FILE[object_uid]FNIX[file_name]
# NOTE: The file-name generation mechanism used here is
# used in other parts of the application. Make sure to
# change those parts as well when you make change the following two lines.
new_fname = filename.translate(None, " ?.!/\;:-{}[]()|~^`")
uid = context.UID() + "FILE" + str(DateTime().millis()) + "FNIX" + new_fname
# Try to create the file object
container.invokeFactory("File", uid, icon=fileobj["icon"])
obj_newObject = container[uid]
obj_newObject._setUID( uid )
obj_newObject.setTitle( fileobj["filename"])
obj_newObject.setDescription("Description of file " + fileobj["filename"])
obj_newObject.setFile( fileobj["file"])
obj_newObject.setContentType( fileobj["content_type"])
obj_newObject.reindexObject()
# Convert the file
converter = Converter( obj_newObject )
converter()
print "Done converting RepositoryItem File object."
resultsurl = ("%s/%s/%s/view#document/p1") %(context.portal_url(), storage_folder, uid)
return resultsurl
else:
print "ERROR: collective.documentviewer is not installed. Unable to view file."
return None
parser.add_argument('--commit',
dest='commit',
type=bool,
default=False,
help='Some scripts care about site id')
args, _ = parser.parse_known_args()
site_id = args.siteid
site = app[site_id] # noqa
setSite(site)
user = app.acl_users.getUser('admin') # noqa
newSecurityManager(None, user.__of__(app.acl_users)) # noqa
gsettings = GlobalSettings(site)
stats = {'keep': 0, 'remove': 0, 'total': 0, 'obfuscated': 0}
obfuscated_paths = {}
obfuscated_uids = []
for brain in site.portal_catalog(portal_type='File'):
stats['total'] += 1
obj = brain.getObject()
settings = Settings(obj)
if settings.obfuscated_filepath:
stats['obfuscated'] += 1
settings.obfuscate_secret
storage_dir = getResourceDirectory(gsettings=gsettings,
settings=settings)
secret_dir = os.path.join(storage_dir, settings.obfuscate_secret)
def _enableAutoConvert(self, enable=True):
"""Enable collective.documentviewer auto_convert."""
gsettings = GlobalSettings(self.portal)
gsettings.auto_convert = enable
gsettings.auto_layout_file_types = CONVERTABLE_TYPES.keys()
return gsettings
