def handle(self, *args, **options): # require user = None if options['user'] is not None: user = get_user_model().objects.get(username=options['user']) filename = options['filename'] if filename.startswith('file://'): filename = filename[7:] if filename.startswith('http://') or filename.startswith('https://'): parsed_url = urlparse(filename) basename = os.path.basename(parsed_url.path) src_file = tempfile.NamedTemporaryFile() req = requests.get(filename, stream=True) chunk_size = 16384 for chunk in req.iter_content(chunk_size): src_file.write(chunk) else: src_file = open(filename, mode='rb') basename = os.path.basename(filename) obj = UploadDoc(uid=str(uuid.uuid1()), user=user, name=options['name'] or basename) obj.save() process_uploaded_file(obj, src_file, basename) for keyword in options['keyword']: obj.keywords.add(Keyword.get(keyword.lower().strip())) obj.save()
def process_file(request: SignalRequest, doc_id: int, filename: str, original_filename: str): """ * uncompress file if needed otherwise copy it * index its content * remove it Remove all if an error occurred :param request: :param doc_id: :param filename: :param original_filename: """ temp_file = None destination_root = None doc = None # noinspection PyBroadException try: temp_file = open(filename, 'rb') doc = UploadDoc.query(request).get(pk=doc_id) destination_root = os.path.join(settings.MEDIA_ROOT, 'docs', doc.uid[0:2], doc.uid) process_uploaded_file(doc, temp_file, original_filename=original_filename, destination_root=destination_root) call('df.messages.info', request, sharing=SESSION, html=_('%(name)s has been uploaded and indexed') % {'name': doc.name}) except Exception as e: if destination_root and os.path.isdir(destination_root): shutil.rmtree(destination_root) UploadDoc.query(request).filter(pk=doc_id).delete() if doc: call('df.messages.error', request, sharing=SESSION, html=_('An error happened during the processing of %(name)s: %(error)s') % {'name': doc.name, 'error': str(e)}) else: call('df.messages.error', request, sharing=SESSION, html=_('Unable to process query')) finally: if temp_file: temp_file.close() os.remove(filename)
def upload_doc_api(request): user = request.user if request.user.is_authenticated else None if user is None: return HttpResponse(_('You must be logged to upload files.\n'), status=401) elif request.method != 'POST': return HttpResponse(_('Only POST requests are allowed.\n'), status=400) form = UploadApiForm(request.GET) if not form.is_valid(): return HttpResponse(_('You must supply filename, name and keywords in your query.\n'), status=400) tmp_file = tempfile.NamedTemporaryFile(mode='wb', dir=settings.FILE_UPLOAD_TEMP_DIR, delete=False) c = False chunk = request.read(32768) while chunk: tmp_file.write(chunk) c = True chunk = request.read(32768) tmp_file.flush() if not c: os.remove(tmp_file.name) return HttpResponse(_('Empty file. You must POST a valid file.\n'), status=400) # ok, we have the tmp file existing_objs = list(UploadDoc.query(request).filter(name=form.cleaned_data['name'])[0:1]) if existing_objs: doc = existing_objs[0] doc.keywords.clear() else: doc = UploadDoc(uid=str(uuid.uuid1()), name=form.cleaned_data['name'], user=user) doc.save() for keyword in strip_split(form.cleaned_data['keywords'].lower()): doc.keywords.add(Keyword.get(keyword)) scall(request, 'updoc.process_file', to=[SERVER], doc_id=doc.id, filename=tmp_file.name, original_filename=os.path.basename(form.cleaned_data['filename'])) return HttpResponse(_('File successfully uploaded. It will be uncompressed and indexed.\n'), status=200)
def edit_doc_keywords(window_info: WindowInfo, doc_id: int, keywords: str): doc = get_object_or_404(UploadDoc.query(window_info), pk=doc_id) doc.keywords.clear() for keyword in [x.strip() for x in keywords.lower().split() if x.strip()]: if keyword: doc.keywords.add(Keyword.get(keyword)) notify(window_info, _('Saved.'), to=WINDOW, timeout=2000)
def process_file(window_info: WindowInfo, doc_id: int, filename: str, original_filename: str): """ * uncompress file if needed otherwise copy it * index its content * remove it Remove all if an error occurred :param window_info: :param doc_id: :param filename: :param original_filename: """ temp_file = None destination_root = None doc = None # noinspection PyBroadException try: temp_file = open(filename, 'rb') doc = UploadDoc.query(window_info).get(pk=doc_id) assert isinstance(doc, UploadDoc) content = _('%(name)s is being processed…') % {'name': doc.name} notify(window_info, content=content, style=NOTIFICATION, level=INFO, timeout=5000, to=WINDOW) destination_root = doc.uncompressed_root process_uploaded_file(doc, temp_file, original_filename=original_filename) content = _('%(name)s has been uploaded and indexed') % {'name': doc.name} notify(window_info, content=content, style=NOTIFICATION, level=SUCCESS, timeout=5000, to=USER) except Exception as e: logger.exception(e) if destination_root and os.path.isdir(destination_root): shutil.rmtree(destination_root) UploadDoc.query(window_info).filter(pk=doc_id).delete() if doc: content = _('An error happened during the processing of %(name)s: %(error)s') % \ {'name': doc.name, 'error': str(e)} notify(window_info, content, to=[USER], style=NOTIFICATION, level=DANGER, timeout=7000) else: content = _('Unable to process query') notify(window_info, content, to=[USER], style=NOTIFICATION, level=DANGER, timeout=7000) finally: if temp_file: temp_file.close() os.remove(filename)
def upload(request): """Index view, displaying and processing a form.""" set_websocket_topics(request) if request.method == 'POST': form = MetadatadUploadForm(request.POST) if form.is_valid(): messages.info(request, _('File successfully uploaded')) obj = get_object_or_404(UploadDoc.query(request), id=form.cleaned_data['pk']) obj.name = form.cleaned_data['name'] for keyword in form.cleaned_data['keywords'].lower().split(): obj.keywords.add(Keyword.get(keyword)) obj.save() return HttpResponseRedirect(reverse('upload')) elif 'pk' in form.cleaned_data: obj = get_object_or_404(UploadDoc.query(request), id=form.cleaned_data['pk']) obj.delete() messages.error(request, _('Unable to upload this file')) else: messages.error(request, _('Unable to upload this file')) else: form = FileUploadForm() template_values = {'form': form, 'title': _('Upload a new file'), 'root_host': settings.SERVER_BASE_URL[:-1]} return TemplateResponse(request, 'updoc/upload.html', template_values)
def upload_doc_progress(request): form = FileUploadForm(request.POST, request.FILES) if not form.is_valid(): raise PermissionDenied uploaded_file = request.FILES['file'] tmp_file = tempfile.NamedTemporaryFile(mode='wb', dir=settings.FILE_UPLOAD_TEMP_DIR, delete=False) chunk = uploaded_file.read(16384) while chunk: tmp_file.write(chunk) chunk = uploaded_file.read(16384) tmp_file.flush() basename = os.path.basename(uploaded_file.name).rpartition('.')[0] if basename.endswith('.tar'): basename = basename[:-4] doc = UploadDoc(name=basename, user=request.user if request.user.is_authenticated else None, uid=str(uuid.uuid1())) doc.save() scall(request, 'updoc.process_file', to=[SERVER], doc_id=doc.id, filename=tmp_file.name, original_filename=uploaded_file.name) # offer a correct name for the newly uploaded document form = MetadatadUploadForm(initial={'pk': doc.pk, 'name': basename, }) template_values = {'form': form, } return TemplateResponse(request, 'updoc/upload_doc_progress.html', template_values)
def my_docs(request): if request.user.is_authenticated: set_websocket_topics(request) user = request.user if request.user.is_authenticated else None if request.method == 'POST' and user and user.has_perm('updoc.add_uploaddoc'): form = UrlRewriteForm(request.POST) if form.is_valid(): RewrittenUrl(user=user, src=form.cleaned_data['src'], dst=form.cleaned_data['dst']).save() messages.info(request, _('URL %(src)s will be rewritten as %(dst)s') % form.cleaned_data) return HttpResponseRedirect(reverse('updoc:my_docs')) else: form = UrlRewriteForm() uploads = UploadDoc.query(request).order_by('-upload_time').select_related() rw_urls = RewrittenUrl.query(request).order_by('src') template_values = {'uploads': uploads, 'title': _('My documents'), 'rw_urls': rw_urls, 'rw_form': form, 'editable': True, 'has_search_results': False, } return TemplateResponse(request, 'updoc/my_docs.html', template_values)
def process_uploaded_file(doc: UploadDoc, temp_file, original_filename: str): """ * Clean previous content if needed * Uncompress content * Index content :param doc: :param temp_file: file descriptor :param original_filename: nom d'origine du fichier """ assert isinstance(doc, UploadDoc) destination_root = doc.uncompressed_root doc_id = doc.id doc_uid = doc.uid UploadDoc.objects.filter(id=doc_id).update(path=destination_root, version=F('version') + 1) doc.path = destination_root doc.version += 1 # first, we clean previous index and data doc.clean_archive() # ok, let's go to decompress destination_root += os.path.sep if original_filename[-7:] in ('.tar.gz', '.tar.xz') or original_filename[-8:] == '.tar.bz2'\ or original_filename[-4:] in ('.tar', '.tbz', '.tgz', '.txz'): tar_file = tarfile.open(name=original_filename, mode='r:*', fileobj=temp_file) names = [name_ for name_ in tar_file.getnames() if os.path.join(doc_uid, name_).startswith(doc_uid)] common_prefix = os.path.sep.join(os.path.commonprefix([name_.split(os.path.sep) for name_ in names])) common_prefix_len = len(common_prefix) for member in tar_file.getmembers(): if not os.path.join(doc_uid, member.name).startswith(doc_uid): continue dst_path = destination_root + member.name[common_prefix_len:] if member.isdir(): os.makedirs(dst_path, mode=0o777, exist_ok=True) elif member.issym(): if not os.path.join(doc_uid, member.linkname).startswith(doc_uid): continue os.symlink(destination_root + member.linkname[common_prefix_len:], dst_path) elif member.isfile(): copy_to_path(tar_file.extractfile(member), dst_path) tar_file.close() elif original_filename[-4:] == '.zip': zip_file = zipfile.ZipFile(temp_file, mode='r') obj_to_extract = [zipped_obj for zipped_obj in zip_file.infolist() if os.path.join(doc_uid, zipped_obj.filename).startswith(doc_uid)] common_prefix = os.path.sep.join(os.path.commonprefix([obj_.filename.split(os.path.sep) for obj_ in obj_to_extract])) common_prefix_len = len(common_prefix) for zipped_obj in obj_to_extract: if zipped_obj.filename[-1:] == os.path.sep: continue copy_to_path(zip_file.open(zipped_obj.filename), destination_root + zipped_obj.filename[common_prefix_len:]) zip_file.close() else: copy_to_path(temp_file, os.path.join(destination_root, original_filename)) # remove unwanted files clean_archive(destination_root) # index index_archive(doc_id, destination_root) # prepare docset Docset(doc).prepare() zip_archive(doc) return doc
def edit_doc_name(window_info: WindowInfo, doc_id: int, name: str): UploadDoc.query(window_info).filter(pk=doc_id).update(name=name) notify(window_info, _('Saved.'), to=WINDOW, timeout=2000)
def delete_doc_confirm(window_info: WindowInfo, doc_id: int): doc = get_object_or_404(UploadDoc.query(window_info), pk=doc_id) template_values = {'name': doc.name, 'doc_id': doc_id, } html = render_to_string('updoc/delete_doc_confirm.html', template_values) modal_show(window_info, html)
def delete_doc(request, doc_id): obj = get_object_or_404(UploadDoc.query(request), id=doc_id) name = obj.name scall(request, 'updoc.delete_file', to=[SERVER], doc_id=doc_id) messages.info(request, _('%(doc)s will be quickly deleted') % {'doc': name}) return HttpResponseRedirect(reverse('updoc:my_docs'))
def edit_doc_keywords(request: SignalRequest, doc_id: int, keywords: str): doc = get_object_or_404(UploadDoc.query(request), pk=doc_id) doc.keywords.clear() for keyword in [x.strip() for x in keywords.lower().split() if x.strip()]: if keyword: doc.keywords.add(Keyword.get(keyword))
def edit_doc_name(request: SignalRequest, doc_id: int, name: str): UploadDoc.query(request).filter(pk=doc_id).update(name=name)
def delete_doc_confirm(request: SignalRequest, doc_id: int): doc = get_object_or_404(UploadDoc.query(request), pk=doc_id) template_values = {'name': doc.name, 'doc_id': doc_id, } html = render_to_string('updoc/delete_doc_confirm.html', template_values) return [{'signal': 'df.modal.show', 'options': {'html': html, }, }]