def test_delete_pages(self): # Create a document with two pages src_file_path = os.path.join(BASE_DIR, "data", "berlin.pdf") with open(src_file_path, 'rb') as fp: src_file = fp.read() init_kwargs = {'payload': src_file, 'processor': 'TEST'} apply_kwargs = {'skip_ocr': True, 'name': "berlin.pdf"} self.assertIsNotNone( go_through_pipelines(init_kwargs=init_kwargs, apply_kwargs=apply_kwargs)) doc = Document.objects.get(title="berlin.pdf") self.assertEqual(doc.page_count, 2) # initial version of any document is 0 self.assertEqual(doc.version, 0) doc.delete_pages(page_numbers=[1], skip_migration=True) self.assertEqual(doc.page_count, 1) self.assertEqual(doc.pages.count(), 1) # version should have been incremented self.assertEqual(doc.version, 1)
def test_simple_pipeline_txt(self): payload_data = ''.join(random.choices( string.ascii_uppercase + string.digits, k=100)).encode() payload = b''.join([MAGIC_BYTES[1][1], payload_data]) for processor in PROCESSORS: init_kwargs = self.make_init_kwargs( payload=payload, processor=processor) apply_kwargs = self.make_apply_kwargs() doc = go_through_pipelines(init_kwargs, apply_kwargs) self.assertIsNone(doc)
def test_default_pipeline_jpg(self): file_path = os.path.join( BASE_DIR, "data", "page-1.jpg" ) with open(file_path, 'rb') as f: payload = f.read() for processor in PROCESSORS: init_kwargs = self.make_init_kwargs( payload=payload, processor=processor) apply_kwargs = self.make_apply_kwargs() doc = go_through_pipelines(init_kwargs, apply_kwargs) self.assertIsNotNone(doc)
def test_simple_pipeline_pdf(self): file_path = os.path.join( BASE_DIR, "data", "berlin.pdf" ) with open(file_path, 'rb') as f: payload = f.read() for processor in PROCESSORS: init_kwargs = self.make_init_kwargs( payload=payload, processor=processor) apply_kwargs = self.make_apply_kwargs() doc = go_through_pipelines(init_kwargs, apply_kwargs) self.assertIsNotNone(doc) self.assertEqual(doc.name, 'test_change_name')
def test_import_file(self): src_file_path = os.path.join(BASE_DIR, "data", "berlin.pdf") with open(src_file_path, 'rb') as fp: src_file = fp.read() init_kwargs = {'payload': src_file, 'processor': 'TEST'} apply_kwargs = {'skip_ocr': True, 'name': "berlin.pdf"} self.assertIsNotNone( go_through_pipelines(init_kwargs=init_kwargs, apply_kwargs=apply_kwargs)) self.assertEqual( Document.objects.filter(title="berlin.pdf").count(), 1, "Document berlin.pdf was not created.")
def put(self, request, filename): file_obj = request.data['file'] init_kwargs = {'payload': file_obj, 'processor': WEB} apply_kwargs = { 'user': request.user.username, 'name': filename, 'apply_async': True } doc = go_through_pipelines(init_kwargs=init_kwargs, apply_kwargs=apply_kwargs) if isinstance(doc, Document): serializer = DocumentSerializer(doc) return Response(serializer.data) return Response(status=200)
def import_documents(directory, skip_ocr=False): files = [] if not directory: raise ValueError("Import directory value is None") for entry in os.scandir(directory): if entry.is_file(): with open(entry.path, 'rb') as file_handler: file_hash = blake2b() file_bytes = file_handler.read() file_hash.update(file_bytes) file_tuple = (entry.path, file_hash.digest()) files.append(file_tuple) else: logger.warning( "Skipping %s as it is not a file", entry.path ) if not files: return time.sleep(int(settings.PAPERMERGE_FILES_MIN_UNMODIFIED_DURATION)) for file_path, file_hash in files: with open(file_path, 'rb') as file_handler: file_bytes = file_handler.read() file_hash_new = blake2b() file_hash_new.update(file_bytes) if not file_hash == file_hash_new.digest(): continue # File has not been modified and can be consumed basename = os.path.basename(file_path) init_kwargs = {'payload': file_bytes, 'processor': LOCAL} apply_kwargs = {'user': None, 'name': basename, 'skip_ocr': skip_ocr } doc = go_through_pipelines(init_kwargs, apply_kwargs) if doc is not None: os.remove(file_path)
def read_email_message(message, user=None, skip_ocr=False): """ message is an instance of python's module email.message """ ingested = False for part in message.iter_attachments(): try: payload = part.get_content() except KeyError: continue init_kwargs = {'payload': payload, 'processor': IMAP} apply_kwargs = { 'user': user, 'name': part.get_filename(), 'skip_ocr': skip_ocr } doc = go_through_pipelines(init_kwargs, apply_kwargs) if doc is not None and not ingested: ingested = True return ingested
def upload(request): """ To understand returned value, have a look at papermerge.core.views.decorators.json_reponse decorator """ files = request.FILES.getlist('file') if not files: logger.warning("POST request.FILES is empty. Forgot adding file?") return "Missing input file", 400 if len(files) > 1: msg = "More then one files per ajax? how come?" logger.warning(msg) return msg, 400 f = files[0] logger.debug("upload for f=%s user=%s", f, request.user) user = request.user parent_id = request.POST.get('parent', "-1") parent_id = filter_node_id(parent_id) lang = request.POST.get('language') notes = request.POST.get('notes') init_kwargs = {'payload': f, 'processor': WEB} apply_kwargs = { 'user': user, 'name': f.name, 'parent': parent_id, 'lang': lang, 'notes': notes, 'apply_async': True } doc = go_through_pipelines(init_kwargs, apply_kwargs) if not doc: status = 400 msg = _("File type not supported." " Only pdf, tiff, png, jpeg files are supported") return msg, status # after each upload return a json object with # following fields: # # - title # - preview_url # - doc_id # - action_url -> needed for renaming/deleting selected item # # with that info a new thumbnail will be created. preview_url = reverse('core:preview', args=(doc.id, 200, 1)) result = { 'title': doc.title, 'doc_id': doc.id, 'action_url': "", 'preview_url': preview_url } return result