def migrate_old_storage(context): catalog = getToolByName(context, 'portal_catalog') portal = getSite() gsettings = GlobalSettings(portal) for brain in catalog(object_provides=IFileContent.__identifier__): file = brain.getObject() if file.getLayout() == 'documentviewer': settings = Settings(file) if settings.storage_version == 1: if settings.storage_type == 'File': current_location = storage.getResourceDirectory( gsettings=gsettings, settings=settings) if not exists(current_location): raise Exception( "oops, can't find storage location %s" % ( current_location)) settings.storage_version = STORAGE_VERSION new_location = storage.getResourceDirectory( gsettings=gsettings, settings=settings) # only make base mkdir_p(os.path.sep.join( new_location.split(os.path.sep)[:-1])) shutil.move(current_location, new_location) # need to commit these eagerly since a failed # migration could leave some migrated wrong transaction.commit() else: settings.storage_version = STORAGE_VERSION
def remove(self): if self.request.get('REQUEST_METHOD', 'POST') and \ self.request.form.get('form.action.remove', '') == 'Remove': authenticator = getMultiAdapter((self.context, self.request), name=u"authenticator") if not authenticator.verify(): raise Unauthorized # find the job sitepath = self.context.getPhysicalPath() async = getUtility(IAsyncService) queue = async.getQueues()[''] objpath = self.request.form.get('path') object = self.context.restrictedTraverse(str(objpath), None) if object is None: return self.redirect() objpath = object.getPhysicalPath() jobs = [job for job in queue] for job in jobs: if isConversion(job, sitepath) and \ job.args[0] == objpath: try: queue.remove(job) settings = Settings(object) settings.converting = False except LookupError: pass return self.redirect() return self.redirect()
def convert_all(context): catalog = getToolByName(context, 'portal_catalog') portal = getSite() gsettings = GlobalSettings(portal) for brain in catalog(object_provides=IFileContent.__identifier__): file = brain.getObject() if not allowedDocumentType(file, gsettings.auto_layout_file_types): continue # let's not switch to the document viewer view # until the document is converted. The conversion # process will check if the layout is set correctly. if file.getLayout() != 'documentviewer': settings = Settings(file) settings.last_updated = DateTime('1999/01/01').ISO8601() queueJob(file) else: settings = Settings(file) # also convert if there was an error. if settings.successfully_converted == False: settings.last_updated = DateTime('1999/01/01').ISO8601() settings.filehash = '' queueJob(file)
def handle_workflow_change(obj, event): settings = Settings(obj) site = getPortal(obj) gsettings = GlobalSettings(site) if not gsettings.storage_obfuscate or \ settings.storage_type != 'File': return for perm in obj.rolesOfPermission("View"): if perm['name'] == 'Anonymous' and perm["selected"] != "": # anon can now view, move it to normal storage_dir = storage_dir = storage.getResourceDirectory( gsettings=gsettings, settings=settings) secret_dir = os.path.join(storage_dir, settings.obfuscate_secret) if not os.path.exists(secret_dir): # already public, oops return for folder in os.listdir(secret_dir): path = os.path.join(secret_dir, folder) newpath = os.path.join(storage_dir, folder) shutil.move(path, newpath) shutil.rmtree(secret_dir) settings.obfuscated_filepath = False return # if we made it here, the item might have been switched back # to being unpublished. Let's just get the converter object # and re-move it converter = Converter(object) converter.handleFileObfuscation()
def remove(self): if self.request.get('REQUEST_METHOD', 'POST') and \ self.request.form.get('form.action.remove', '') == 'Remove': authenticator = getMultiAdapter((self.context, self.request), name=u"authenticator") if not authenticator.verify(): raise Unauthorized # find the job sitepath = self.context.getPhysicalPath() async = getUtility(IAsyncService) queue = async .getQueues()[''] objpath = self.request.form.get('path') obj = self.context.restrictedTraverse(str(objpath), None) if obj is None: return self.redirect() objpath = obj.getPhysicalPath() jobs = [job for job in queue] for job in jobs: if isConversion(job, sitepath) and \ job.args[0] == objpath: try: queue.remove(job) settings = Settings(obj) settings.converting = False except LookupError: pass return self.redirect() return self.redirect()
def migrate_old_storage(context): catalog = getToolByName(context, 'portal_catalog') portal = getSite() gsettings = GlobalSettings(portal) for brain in catalog(object_provides=IFileContent.__identifier__): file = brain.getObject() if file.getLayout() == 'documentviewer': settings = Settings(file) if settings.storage_version == 1: if settings.storage_type == 'File': current_location = storage.getResourceDirectory( gsettings=gsettings, settings=settings) if not exists(current_location): raise Exception( "oops, can't find storage location %s" % (current_location)) settings.storage_version = STORAGE_VERSION new_location = storage.getResourceDirectory( gsettings=gsettings, settings=settings) # only make base mkdir_p( os.path.sep.join(new_location.split(os.path.sep)[:-1])) shutil.move(current_location, new_location) # need to commit these eagerly since a failed # migration could leave some migrated wrong transaction.commit() else: settings.storage_version = STORAGE_VERSION
def handle_workflow_change(object, event): settings = Settings(object) site = getPortal(object) gsettings = GlobalSettings(site) if not gsettings.storage_obfuscate or \ settings.storage_type != 'File': return for perm in object.rolesOfPermission("View"): if perm['name'] == 'Anonymous' and perm["selected"] != "": # anon can now view, move it to normal storage_dir = storage_dir = storage.getResourceDirectory( gsettings=gsettings, settings=settings) secret_dir = os.path.join(storage_dir, settings.obfuscate_secret) if not os.path.exists(secret_dir): # already public, oops return for folder in os.listdir(secret_dir): path = os.path.join(secret_dir, folder) newpath = os.path.join(storage_dir, folder) shutil.move(path, newpath) shutil.rmtree(secret_dir) settings.obfuscated_filepath = False return # if we made it here, the item might have been switched back # to being unpublished. Let's just get the converter object # and re-move it converter = Converter(object) converter.handleFileObfuscation()
def __call__(self): req = self.request settings = Settings(self.context) annotations = settings.annotations if annotations is None: annotations = PersistentDict() settings.annotations = annotations sections = settings.sections if sections is None: sections = PersistentList() settings.sections = sections action = req.form['action'] if action == 'addannotation': page = int(req.form['page']) if page not in annotations: annotations[page] = PersistentList() pageann = annotations[page] data = { "id": random.randint(1, 9999999), "coord": req.form['coord'], "title": req.form.get('title', ''), "content": req.form.get('content', '')} pageann.append(data) return json.dumps(data) elif action == 'removeannotation': page = int(req.form['page']) if page in annotations: ann_id = int(req.form['id']) found = False annotations = annotations[page] for ann in annotations: if ann['id'] == ann_id: found = ann break if found: annotations.remove(found) elif action == 'addsection': data = { 'page': req.form['page'], 'title': req.form['title'] } sections.append(data) return json.dumps(data) elif action == 'removesection': data = { 'page': req.form['page'], 'title': req.form['title'] } if data in sections: sections.remove(data)
def get_thumb(self, obj): if not obj: return None resource_rel = storage.getResourceRelURL(obj=obj) if self.resource_url: dvpdffiles = '%s/%s' % (self.resource_url.rstrip('/'), resource_rel) else: dvpdffiles = '%s/%s' % (self.portal_url, resource_rel) if obj.portal_type == 'File': settings = Settings(obj) if settings.successfully_converted: image_format = settings.pdf_image_format if not image_format: image_format = self.global_settings.pdf_image_format return '%s/small/%s_1.%s' % (dvpdffiles, self.dump_path, image_format) else: return '%s/images/pdf.png' % (self.static_url) elif obj.portal_type == 'Image': url = obj.absolute_url() return '%s/image_thumb' % url
def clean_folder(self, catalog, storage_loc): if not os.path.isdir(storage_loc): return 0 count = 0 for foldername in os.listdir(storage_loc): if len(foldername) == 1: # we're in a container, check inside count += self.clean_folder( catalog, os.path.join(storage_loc, foldername)) else: # foldername should be file uid brains = catalog(UID=foldername) folderpath = os.path.join(storage_loc, foldername) if len(brains) == 0: shutil.rmtree(folderpath) count += 1 else: obj = brains[0].getObject() settings = Settings(obj) if obj.getLayout() != 'documentviewer': if not settings.converting: shutil.rmtree(folderpath) count += 1 # also delete settings annotations = IAnnotations(obj) data = annotations.get('collective.documentviewer', None) if data: del annotations['collective.documentviewer'] elif settings.storage_type == 'Blob': shutil.rmtree(folderpath) count += 1 return count
def __call__(self): """ Redirects to the url for the rendered PDF. We need to redirect, because the PDF can be stored on FS, instead of ZODB. """ try: site = api.portal.get() except exc.CannotGetPortalError: raise NotFound settings = Settings(self.context) global_settings = GlobalSettings(site) portal_url = getMultiAdapter((self.context, self.request), name="plone_portal_state").portal_url() resource_url = global_settings.override_base_resource_url rel_url = storage.getResourceRelURL(gsettings=global_settings, settings=settings) if resource_url: dvpdffiles = '%s/%s' % (resource_url.rstrip('/'), rel_url) else: dvpdffiles = '%s/%s' % (portal_url, rel_url) url = '%s/pdf/dump.pdf' % dvpdffiles self.request.response.redirect(url)
def __call__(self): self.site = getPortal(self.context) self.settings = Settings(self.context) self.global_settings = GlobalSettings(self.site) self.portal_url = getMultiAdapter( (self.context, self.request), name="plone_portal_state").portal_url() self.dvstatic = "%s/++resource++dv.resources" % (self.portal_url) resource_url = self.global_settings.override_base_resource_url rel_url = storage.getResourceRelURL(gsettings=self.global_settings, settings=self.settings) if resource_url: self.dvpdffiles = '%s/%s' % (resource_url.rstrip('/'), rel_url) else: self.dvpdffiles = '%s/%s' % (self.portal_url, rel_url) utils = getToolByName(self.context, 'plone_utils') msg = None self.enabled = True if allowedDocumentType(self.context, self.global_settings.auto_layout_file_types): if not self.installed: msg = _( "Since you do not have docsplit installed on this " "system, we can not render the pages of this document.") if self.settings.converting is not None and \ self.settings.converting: if self.settings.successfully_converted: # there is a version that is already converted, show it. self.enabled = True else: msg = _("The document is currently being converted to the " "Document Viewer view.") self.enabled = False elif self.settings.successfully_converted is not None and \ not self.settings.successfully_converted: msg = _("There was an error trying to convert the document. " "Maybe the document is encrypted, corrupt or " "malformed? Check log for details.") self.enabled = False elif self.settings.successfully_converted is None: # must have just switched to this view msg = _("This document is not yet converted to document " "viewer. Please click the `Document Viewer Convert` " "button in the actions menu to convert.") self.enabled = False else: self.enabled = False msg = _("The file is not a supported document type. " "Your type may be supported. Check out the document " "viewer configuration settings.") mtool = getToolByName(self.context, 'portal_membership') self.can_modify = mtool.checkPermission('cmf.ModifyPortalContent', self.context) if msg and self.can_modify: utils.addPortalMessage(_(msg)) return self.index()
def render_attachment_preview(self, attachment): sm = getSecurityManager() if not sm.checkPermission(permissions.View, self.context): raise Unauthorized r = self.request.response settings = Settings(attachment) if self.preview_type not in ('large', 'normal', 'small'): self.preview_type = 'small' if self.page is None: self.page = 1 filepath = u'%s/dump_%s.%s' % (self.preview_type, self.page, settings.pdf_image_format) blob = settings.blob_files[filepath] blobfi = openBlob(blob) length = os.fstat(blobfi.fileno()).st_size blobfi.close() ext = os.path.splitext(os.path.normcase(filepath))[1][1:] if ext == 'txt': ct = 'text/plain' else: ct = 'image/%s' % ext r.setHeader('Content-Type', ct) r.setHeader('Last-Modified', rfc1123_date(self.context._p_mtime)) r.setHeader('Accept-Ranges', 'bytes') r.setHeader("Content-Length", length) request_range = handleRequestRange(self.context, length, self.request, self.request.response) return BlobStreamIterator(blob, **request_range)
def test_publish_unpublish_again_works_with_obfuscation(self): gsettings = GlobalSettings(self.portal) _dir = mkdtemp() gsettings.storage_location = _dir gsettings.storage_type = 'File' gsettings.storage_obfuscate = True fi = self.createFile('test.pdf') uid = fi.UID() fi.reindexObject() notify(ObjectInitializedEvent(fi)) settings = Settings(fi) self.assertTrue( exists(join(_dir, uid[0], uid[1], uid, settings.obfuscate_secret))) self.assertTrue( listdir(join(_dir, uid[0], uid[1], uid, settings.obfuscate_secret)) > 3) # publish now workflowTool = getToolByName(fi, 'portal_workflow') workflowTool.doActionFor(fi, 'publish') self.assertTrue(not exists( join(_dir, uid[0], uid[1], uid, settings.obfuscate_secret))) workflowTool.doActionFor(fi, 'retract') self.assertTrue( exists(join(_dir, uid[0], uid[1], uid, settings.obfuscate_secret))) self.assertTrue( listdir(join(_dir, uid[0], uid[1], uid, settings.obfuscate_secret)) > 3)
def test_get_correct_rel_url_for_old_storage(self): fi = self.createFile('test.pdf') settings = Settings(fi) del settings._metadata['storage_version'] uid = fi.UID() self.assertEquals(storage.getResourceRelURL(obj=fi), '@@dvpdffiles/%s' % uid)
def _isSuccessfullyConverted(self, fi): ''' Check if the given p_fi was successfully converted ''' # make sure conversion was successfull settings = Settings(fi) return settings.successfully_converted
def test_set_storage_version(self): gsettings = GlobalSettings(self.portal) _dir = mkdtemp() gsettings.storage_location = _dir fi = self.createFile('test.pdf') settings = Settings(fi) self.assertEquals(settings.storage_version, STORAGE_VERSION)
def test_converts(self): fi = self.createFile('test.pdf') settings = Settings(fi) self.assertEqual(settings.successfully_converted, None) notify(ObjectInitializedEvent(fi)) self.assertEqual(settings.successfully_converted, True) self.assertEqual(settings.num_pages, 1)
def __call__(self): settings = Settings(self.context) catalog = settings.catalog if catalog: query = self.request.form.get('q') results = catalog.query(Contains('text', query)) return json.dumps({"results": list(results[1]), "query": query}) return json.dumps({"results": [], "query": query})
def test_sets_filehash(self): gsettings = GlobalSettings(self.portal) gsettings.auto_select_layout = True gsettings.auto_layout_file_types = ['ppt'] fi = self.createFile('test.odp') settings = Settings(fi) self.assertTrue(settings.filehash is not None)
def test_get_correct_rel_url_for_old_storage_with_resource_url(self): fi = self.createFile('test.pdf') settings = Settings(fi) gsettings = GlobalSettings(self.portal) gsettings.override_base_resource_url = 'http://foobar.com' del settings._metadata['storage_version'] uid = fi.UID() self.assertEquals(storage.getResourceRelURL(obj=fi), '%s' % uid)
def get_file_thumbs_urls(self, attachment): ''' thumbs provided by c.dv. Only returning the front page. ''' settings = Settings(attachment) if settings.successfully_converted is not True: return None return [attachment.absolute_url() + '/small']
def test_sets_filehash(self): fi = self.createFile('test.odp') gsettings = GlobalSettings(self.portal) gsettings.auto_select_layout = True gsettings.auto_layout_file_types = ['ppt'] notify(ObjectInitializedEvent(fi)) settings = Settings(fi) self.assertTrue(settings.filehash is not None)
def convert_all(context): catalog = getToolByName(context, 'portal_catalog') portal = getSite() gsettings = GlobalSettings(portal) for brain in catalog(object_provides=OBJECT_PROVIDES): file_item = brain.getObject() if not allowedDocumentType(file_item, gsettings.auto_layout_file_types): continue # let's not switch to the document viewer view # until the document is converted. The conversion # process will check if the layout is set correctly. if file_item.getLayout() != 'documentviewer': settings = Settings(file_item) settings.last_updated = DateTime('1999/01/01').ISO8601() queueJob(file_item) else: settings = Settings(file_item) # also convert if there was an error. if settings.successfully_converted == False: settings.last_updated = DateTime('1999/01/01').ISO8601() settings.filehash = '' queueJob(file_item)
def test_auto_convert_word(self): gsettings = GlobalSettings(self.portal) gsettings.auto_select_layout = True gsettings.auto_layout_file_types = ['word'] fi = self.createFile('test.doc') settings = Settings(fi) self.assertEqual(settings.successfully_converted, True) self.assertEqual(settings.num_pages, 2)
def __init__(self, context): self.context = aq_inner(context) self.settings = Settings(self.context) self.filehash = None self.blob_filepath = None self.gsettings = GlobalSettings(getPortal(context)) self.storage_dir = self.get_storage_dir() self.doc_type = getDocumentType(self.context, self.gsettings.auto_layout_file_types)
def test_retrieve_correct_resource_location_old_storage(self): gsettings = GlobalSettings(self.portal) _dir = mkdtemp() gsettings.storage_location = _dir fi = self.createFile('test.pdf') settings = Settings(fi) del settings._metadata['storage_version'] self.assertEquals(storage.getResourceDirectory(obj=fi), join(_dir, fi.UID()))
def test_auto_convert_powerpoint(self): fi = self.createFile('test.odp') gsettings = GlobalSettings(self.portal) gsettings.auto_select_layout = True gsettings.auto_layout_file_types = ['ppt'] notify(ObjectInitializedEvent(fi)) settings = Settings(fi) self.assertEqual(settings.successfully_converted, True) self.assertEqual(settings.num_pages, 1)
def get_preview_urls(obj, scale='normal', with_timestamp=False): """Convenience method to get URLs of image previews as these are most frequently used :param obj: The Plone content object to get preview URLs for :type obj: A Plone content object :param with_timestamp: If True add a timestamp to the URLs :type with_timestamp: bool :param scale: The Plone image scale to get preview images at :type scale: str :return: List of preview image absolute URLs :rtype: list """ dv_data = _get_dv_data(obj) scale = _backward_map(scale) settings = Settings(obj) if not settings or settings.blob_files is None: return [fallback_image_url(obj)] previews = [] ext = settings.pdf_image_format for i in range(len(settings.blob_files)): preview = '%s/dump_%s.%s' % (scale, str(i + 1), ext) if settings.blob_files.get(preview, False): previews.append(settings.blob_files[preview]) number_of_previews = dv_data['pages'] # If there aren't any previews, return the placeholder url if number_of_previews < 1: return [fallback_image_url(obj)] urls = [ dv_data['resources']['page']['image'].format(size=scale, page=page) for page in range(1, len(previews) + 1) ] if with_timestamp: try: timestamp = Settings(obj).last_updated except: log.exception('Cannot get timestamp from %r', obj) timestamp = datetime.now().isoformat() qs = '?%s' % urlencode({'t': timestamp}) urls = [(url + qs) for url in urls] return urls
def successfully_converted(obj): """ Check if object could be converted :param obj: The Plone object to get previews for :type obj: A Plone content object :return: True if successfully converted, False if conversion failed :rtype: boolean """ settings = Settings(obj) return settings.successfully_converted
def converting(obj): """ Check if object is currently being converted :param obj: The Plone object to get previews for :type obj: A Plone content object :return: True if converting, False if no longer converting :rtype: boolean """ settings = Settings(obj) return settings.converting
def handle_file_delete(obj, event): if obj.portal_type == 'Image': return # need to remove files if stored in file system settings = Settings(obj) if settings.storage_type == 'File': storage_directory = getResourceDirectory(obj=obj) if os.path.exists(storage_directory): shutil.rmtree(storage_directory)
def __init__(self, context): self.context = aq_inner(context) self.settings = Settings(self.context) fw = IFileWrapper(self.context) self.blob = fw.blob self.initialize_blob_filepath() self.filehash = None self.gsettings = GlobalSettings(api.portal.get()) self.storage_dir = self.get_storage_dir() self.doc_type = getDocumentType(self.context, self.gsettings.auto_layout_file_types)
def __call__(self): """ - handle queuing - csrf protection - async - queue position """ mtool = getToolByName(self.context, 'portal_membership') self.manager = mtool.checkPermission('cmf.ManagePortal', self.context) self.async_installed = asyncInstalled() self.converting = False if self.enabled(): req = self.request if req.get('REQUEST_METHOD', 'POST') and \ 'form.action.queue' in req.form.keys(): authenticator = getMultiAdapter((self.context, self.request), name=u"authenticator") if not authenticator.verify(): raise Unauthorized settings = Settings(self.context) settings.last_updated = DateTime('1999/01/01').ISO8601() settings.filehash = '--foobar--' queueJob(self.context) self.converting = True if self.async_installed: self.position = JobRunner(self.context).find_position() queueJob(self.context) else: return self.request.response.redirect( self.context.absolute_url() + '/view') else: if self.async_installed: self.position = JobRunner(self.context).find_position() if self.position > -1: self.converting = True return super(Convert, self).__call__() self.request.response.redirect(self.context.absolute_url() + '/view')
def queue_it(self): self.async.queueJobInQueue(self.queue, (QUOTA_NAME,), runConversion, self.object) settings = Settings(self.object) settings.converting = True