def _soffice_process(tempfile_path, filename, md5_hash, process_type): '''create processed file,upload to s3, store ref''' #libre office requires invidual environs to run multiple instances #make empty file named to hash for unique we haz already. loffice_environ_path = os.path.join('/tmp', md5_hash) try: os.makedirs(loffice_environ_path) except FileExistsError: pass s = filename.split('.') child_name = '.'.join(s[:-1]) + '.' + process_type extension = s[-1] outpath = os.path.join('/tmp', child_name) #t1 = time.time() try: os.system('/usr/bin/soffice -env:UserInstallation=file://%s \ --headless --convert-to %s %s --outdir %s' \ % (loffice_environ_path, process_type, tempfile_path, '/tmp')) except: raise HTTPExceptions.UNPROCESSABLE_ENTITY s3 = S3(settings.AWS_ANNOTATIONS_BUCKET) saved_file = open(outpath, 'rb') s3.save_to_bucket(child_name, saved_file) #save ref to db ref = FileUpload(filename=child_name, md5_hash=md5_hash, extension=extension, is_original=False) ref.save() cleanup_temp_file(child_name) cleanup_temp_file(filename) #remove environment file try: shutil.rmtree(loffice_environ_path) except: #shrug pass return child_name
def ocr_pdf(filename, parent_id, md5_hash, force_flag): if not os.path.exists('/tmp/ocr_clients'): os.makedirs('/tmp/ocr_clients') lockfile = os.path.join('/tmp/ocr_clients', md5_hash) try: #prevent too many heavy ocr processes from running at once current_process_count = len(os.listdir('/tmp/ocr_clients')) if current_process_count >= int(settings.MAX_SIM_OCR_PROCESSES): raise MaxProcessesExceededError() #add to current process count with file try: f = open(lockfile, 'x') f.close() except FileExistsError: raise FileInProcessError() input_path = os.path.join('/tmp', filename) #download file and save s3 = S3(settings.AWS_OCR_BUCKET) file_obj = s3.download_fileobj_from_bucket(filename) #file_obj.save(input_path) with open (input_path, 'wb') as tmpfile: tmpfile.write(file_obj.getbuffer()) basename = '.'.join(filename.split('.')[:-1]) if force_flag: processed_filename = basename + '_ocr_force.pdf' force_flag = True else: processed_filename = basename + '_ocr.pdf' force_flag = False output_path = os.path.join('/tmp', processed_filename) if force_flag: f = '--force-ocr' else: f = '' cmd = '/usr/bin/ocrmypdf {} {} {}'.format(f, input_path, output_path) rslt = subprocess.check_output(cmd, shell=True) #save to s3 with open(output_path, 'rb') as file_: s3.save_to_bucket(processed_filename, file_) file_.seek(0) hash_ = md5(file_.read()).hexdigest() #record to db ref = OCRUpload(filename=processed_filename, md5_hash=hash_, is_original=False, is_forced=force_flag, parent_id=parent_id) ref.save() #remove from process count os.remove(lockfile) cleanup_temp_file(filename) cleanup_temp_file(processed_filename) except Exception as e: try: os.remove(os.path.join('/tmp/ocr_clients', md5_hash)) cleanup_temp_file(filename) cleanup_temp_file(processed_filename) except: pass raise e
def upload(request): if request.method == 'POST': file_ = request.FILES.get('pdf-file') processing_error = None if file_ is None: raise HTTPExceptions.NOT_ACCEPTABLE #Error code 406 filename = file_.name if not filename or len(filename) < 3 or not '.' in filename: raise SuspiciousFileOperation('improper file name') filename = sanitize(filename) filename = filename.replace("'", '').replace('"', '') filename = re.sub(r"[\(,\),\s]+", "-", filename) temp = filename.split('.') basename = '.'.join(temp[:-1]) extension = temp[-1] if not extension in ('pdf', 'PDF'): raise SuspiciousFileOperation('improper file type') basename = basename[:60] new_filename = '{0}-{1}.{2}'.format(basename, randword(5), extension) #save to /tmp md5_hash, tempfile_path = save_temp_file(new_filename, file_) #file already exists in system? existing_name = check_ocr_file_exists(md5_hash) #already_has_text? if check_pdf_has_text(new_filename): processing_error = 'This PDF already has text. Use the "Force OCR" button to overwrite text with a fresh OCR if desired. If file was OCRd on previous upload those results will be provided' if not existing_name: already_exists = False #upload original to S3 s3 = S3(settings.AWS_OCR_BUCKET) saved_file = open(tempfile_path, 'rb') s3.save_to_bucket(new_filename, saved_file) ref = OCRUpload(filename=new_filename, md5_hash=md5_hash, is_original=True) ref.save() cleanup_temp_file(new_filename) else: already_exists = True new_filename = existing_name cleanup_temp_file(new_filename) data = { 'file_info': { 'filename': filename, 'size': file_.size, 'new_filename': new_filename, 'processing_error': processing_error, 'tempfile_path': tempfile_path, 'already_exists': already_exists, 'md5_hash': md5_hash } } return JsonResponse(data) return HttpResponseNotAllowed(['POST,'])
def upload(request): filename = "" if request.method == 'POST': file_ = request.FILES['file'] filename = file_.name if not filename or len(filename) < 3 or not '.' in filename: raise SuspiciousFileOperation('improper file name') filename = sanitize(filename) filename = filename.replace("'", '').replace('"', '') filename = re.sub(r"[\(,\),\s]+", "-", filename) temp = filename.split('.') basename = '.'.join(temp[:-1]) extension = temp[-1] basename = basename[:60] new_filename = '{0}-{1}.{2}'.format(basename, randword(5), extension) #save file to disk temporarily. #later it will be deleted after uploading to s3. md5_hash, tempfile_path = save_temp_file(new_filename, file_) extension = extension.lower() #if file (or processed child) exists, return the name existing_name = check_file_exists(md5_hash) if existing_name: cleanup_temp_file(new_filename) return HttpResponse(existing_name) #transform process if needed process_to_file_type = False if extension in ['doc', 'docx', 'odt', 'ott', 'rtf', 'odp', 'ppt', 'pptx']: process_to_file_type = 'pdf' if extension in ['xls', 'xlsx', 'ods']: process_to_file_type = 'csv' if process_to_file_type: child_name = _soffice_process( tempfile_path, new_filename, md5_hash, process_to_file_type) if child_name: cleanup_temp_file(child_name) return HttpResponse(child_name) else: cleanup_temp_file(child_name) raise HTTPExceptions.UNPROCESSABLE_ENTITY if extension == 'pdf': #check if is an image pdf or if it has text if not check_pdf_has_text(new_filename): cleanup_temp_file(new_filename) raise HTTPExceptions.NOT_ACCEPTABLE #Error code 406 #upload to cloud s3 = S3(settings.AWS_ANNOTATIONS_BUCKET) saved_file = open(tempfile_path, 'rb') s3.save_to_bucket(new_filename, saved_file) #save ref to db ref = FileUpload(filename=new_filename, md5_hash=md5_hash, extension=extension, is_original=True) ref.save() cleanup_temp_file(new_filename) return HttpResponse(new_filename) return HttpResponseNotAllowed(['POST,'])