def handle(self, *args, **options):
# require
user = None
if options['user'] is not None:
user = get_user_model().objects.get(username=options['user'])
filename = options['filename']
if filename.startswith('file://'):
filename = filename[7:]
if filename.startswith('http://') or filename.startswith('https://'):
parsed_url = urlparse(filename)
basename = os.path.basename(parsed_url.path)
src_file = tempfile.NamedTemporaryFile()
req = requests.get(filename, stream=True)
chunk_size = 16384
for chunk in req.iter_content(chunk_size):
src_file.write(chunk)
else:
src_file = open(filename, mode='rb')
basename = os.path.basename(filename)
obj = UploadDoc(uid=str(uuid.uuid1()), user=user, name=options['name'] or basename)
obj.save()
process_uploaded_file(obj, src_file, basename)
for keyword in options['keyword']:
obj.keywords.add(Keyword.get(keyword.lower().strip()))
obj.save()
def process_file(request: SignalRequest, doc_id: int, filename: str, original_filename: str):
"""
* uncompress file if needed otherwise copy it
* index its content
* remove it
Remove all if an error occurred
:param request:
:param doc_id:
:param filename:
:param original_filename:
"""
temp_file = None
destination_root = None
doc = None
# noinspection PyBroadException
try:
temp_file = open(filename, 'rb')
doc = UploadDoc.query(request).get(pk=doc_id)
destination_root = os.path.join(settings.MEDIA_ROOT, 'docs', doc.uid[0:2], doc.uid)
process_uploaded_file(doc, temp_file, original_filename=original_filename, destination_root=destination_root)
call('df.messages.info', request, sharing=SESSION, html=_('%(name)s has been uploaded and indexed') % {'name': doc.name})
except Exception as e:
if destination_root and os.path.isdir(destination_root):
shutil.rmtree(destination_root)
UploadDoc.query(request).filter(pk=doc_id).delete()
if doc:
call('df.messages.error', request, sharing=SESSION, html=_('An error happened during the processing of %(name)s: %(error)s') % {'name': doc.name, 'error': str(e)})
else:
call('df.messages.error', request, sharing=SESSION, html=_('Unable to process query'))
finally:
if temp_file:
temp_file.close()
os.remove(filename)
def upload_doc_api(request):
user = request.user if request.user.is_authenticated else None
if user is None:
return HttpResponse(_('You must be logged to upload files.\n'), status=401)
elif request.method != 'POST':
return HttpResponse(_('Only POST requests are allowed.\n'), status=400)
form = UploadApiForm(request.GET)
if not form.is_valid():
return HttpResponse(_('You must supply filename, name and keywords in your query.\n'), status=400)
tmp_file = tempfile.NamedTemporaryFile(mode='wb', dir=settings.FILE_UPLOAD_TEMP_DIR, delete=False)
c = False
chunk = request.read(32768)
while chunk:
tmp_file.write(chunk)
c = True
chunk = request.read(32768)
tmp_file.flush()
if not c:
os.remove(tmp_file.name)
return HttpResponse(_('Empty file. You must POST a valid file.\n'), status=400)
# ok, we have the tmp file
existing_objs = list(UploadDoc.query(request).filter(name=form.cleaned_data['name'])[0:1])
if existing_objs:
doc = existing_objs[0]
doc.keywords.clear()
else:
doc = UploadDoc(uid=str(uuid.uuid1()), name=form.cleaned_data['name'], user=user)
doc.save()
for keyword in strip_split(form.cleaned_data['keywords'].lower()):
doc.keywords.add(Keyword.get(keyword))
scall(request, 'updoc.process_file', to=[SERVER], doc_id=doc.id, filename=tmp_file.name,
original_filename=os.path.basename(form.cleaned_data['filename']))
return HttpResponse(_('File successfully uploaded. It will be uncompressed and indexed.\n'), status=200)
def edit_doc_keywords(window_info: WindowInfo, doc_id: int, keywords: str):
doc = get_object_or_404(UploadDoc.query(window_info), pk=doc_id)
doc.keywords.clear()
for keyword in [x.strip() for x in keywords.lower().split() if x.strip()]:
if keyword:
doc.keywords.add(Keyword.get(keyword))
notify(window_info, _('Saved.'), to=WINDOW, timeout=2000)
def process_file(window_info: WindowInfo, doc_id: int, filename: str, original_filename: str):
"""
* uncompress file if needed otherwise copy it
* index its content
* remove it
Remove all if an error occurred
:param window_info:
:param doc_id:
:param filename:
:param original_filename:
"""
temp_file = None
destination_root = None
doc = None
# noinspection PyBroadException
try:
temp_file = open(filename, 'rb')
doc = UploadDoc.query(window_info).get(pk=doc_id)
assert isinstance(doc, UploadDoc)
content = _('%(name)s is being processed…') % {'name': doc.name}
notify(window_info, content=content, style=NOTIFICATION, level=INFO, timeout=5000, to=WINDOW)
destination_root = doc.uncompressed_root
process_uploaded_file(doc, temp_file, original_filename=original_filename)
content = _('%(name)s has been uploaded and indexed') % {'name': doc.name}
notify(window_info, content=content, style=NOTIFICATION, level=SUCCESS, timeout=5000, to=USER)
except Exception as e:
logger.exception(e)
if destination_root and os.path.isdir(destination_root):
shutil.rmtree(destination_root)
UploadDoc.query(window_info).filter(pk=doc_id).delete()
if doc:
content = _('An error happened during the processing of %(name)s: %(error)s') % \
{'name': doc.name, 'error': str(e)}
notify(window_info, content, to=[USER], style=NOTIFICATION, level=DANGER, timeout=7000)
else:
content = _('Unable to process query')
notify(window_info, content, to=[USER], style=NOTIFICATION, level=DANGER, timeout=7000)
finally:
if temp_file:
temp_file.close()
os.remove(filename)
def upload(request):
"""Index view, displaying and processing a form."""
set_websocket_topics(request)
if request.method == 'POST':
form = MetadatadUploadForm(request.POST)
if form.is_valid():
messages.info(request, _('File successfully uploaded'))
obj = get_object_or_404(UploadDoc.query(request), id=form.cleaned_data['pk'])
obj.name = form.cleaned_data['name']
for keyword in form.cleaned_data['keywords'].lower().split():
obj.keywords.add(Keyword.get(keyword))
obj.save()
return HttpResponseRedirect(reverse('upload'))
elif 'pk' in form.cleaned_data:
obj = get_object_or_404(UploadDoc.query(request), id=form.cleaned_data['pk'])
obj.delete()
messages.error(request, _('Unable to upload this file'))
else:
messages.error(request, _('Unable to upload this file'))
else:
form = FileUploadForm()
template_values = {'form': form, 'title': _('Upload a new file'), 'root_host': settings.SERVER_BASE_URL[:-1]}
return TemplateResponse(request, 'updoc/upload.html', template_values)
def upload_doc_progress(request):
form = FileUploadForm(request.POST, request.FILES)
if not form.is_valid():
raise PermissionDenied
uploaded_file = request.FILES['file']
tmp_file = tempfile.NamedTemporaryFile(mode='wb', dir=settings.FILE_UPLOAD_TEMP_DIR, delete=False)
chunk = uploaded_file.read(16384)
while chunk:
tmp_file.write(chunk)
chunk = uploaded_file.read(16384)
tmp_file.flush()
basename = os.path.basename(uploaded_file.name).rpartition('.')[0]
if basename.endswith('.tar'):
basename = basename[:-4]
doc = UploadDoc(name=basename, user=request.user if request.user.is_authenticated else None,
uid=str(uuid.uuid1()))
doc.save()
scall(request, 'updoc.process_file', to=[SERVER], doc_id=doc.id, filename=tmp_file.name,
original_filename=uploaded_file.name)
# offer a correct name for the newly uploaded document
form = MetadatadUploadForm(initial={'pk': doc.pk, 'name': basename, })
template_values = {'form': form, }
return TemplateResponse(request, 'updoc/upload_doc_progress.html', template_values)
def my_docs(request):
if request.user.is_authenticated:
set_websocket_topics(request)
user = request.user if request.user.is_authenticated else None
if request.method == 'POST' and user and user.has_perm('updoc.add_uploaddoc'):
form = UrlRewriteForm(request.POST)
if form.is_valid():
RewrittenUrl(user=user, src=form.cleaned_data['src'], dst=form.cleaned_data['dst']).save()
messages.info(request, _('URL %(src)s will be rewritten as %(dst)s') % form.cleaned_data)
return HttpResponseRedirect(reverse('updoc:my_docs'))
else:
form = UrlRewriteForm()
uploads = UploadDoc.query(request).order_by('-upload_time').select_related()
rw_urls = RewrittenUrl.query(request).order_by('src')
template_values = {'uploads': uploads, 'title': _('My documents'), 'rw_urls': rw_urls,
'rw_form': form, 'editable': True, 'has_search_results': False, }
return TemplateResponse(request, 'updoc/my_docs.html', template_values)
def process_uploaded_file(doc: UploadDoc, temp_file, original_filename: str):
"""
* Clean previous content if needed
* Uncompress content
* Index content
:param doc:
:param temp_file: file descriptor
:param original_filename: nom d'origine du fichier
"""
assert isinstance(doc, UploadDoc)
destination_root = doc.uncompressed_root
doc_id = doc.id
doc_uid = doc.uid
UploadDoc.objects.filter(id=doc_id).update(path=destination_root, version=F('version') + 1)
doc.path = destination_root
doc.version += 1
# first, we clean previous index and data
doc.clean_archive()
# ok, let's go to decompress
destination_root += os.path.sep
if original_filename[-7:] in ('.tar.gz', '.tar.xz') or original_filename[-8:] == '.tar.bz2'\
or original_filename[-4:] in ('.tar', '.tbz', '.tgz', '.txz'):
tar_file = tarfile.open(name=original_filename, mode='r:*', fileobj=temp_file)
names = [name_ for name_ in tar_file.getnames() if os.path.join(doc_uid, name_).startswith(doc_uid)]
common_prefix = os.path.sep.join(os.path.commonprefix([name_.split(os.path.sep) for name_ in names]))
common_prefix_len = len(common_prefix)
for member in tar_file.getmembers():
if not os.path.join(doc_uid, member.name).startswith(doc_uid):
continue
dst_path = destination_root + member.name[common_prefix_len:]
if member.isdir():
os.makedirs(dst_path, mode=0o777, exist_ok=True)
elif member.issym():
if not os.path.join(doc_uid, member.linkname).startswith(doc_uid):
continue
os.symlink(destination_root + member.linkname[common_prefix_len:], dst_path)
elif member.isfile():
copy_to_path(tar_file.extractfile(member), dst_path)
tar_file.close()
elif original_filename[-4:] == '.zip':
zip_file = zipfile.ZipFile(temp_file, mode='r')
obj_to_extract = [zipped_obj for zipped_obj in zip_file.infolist()
if os.path.join(doc_uid, zipped_obj.filename).startswith(doc_uid)]
common_prefix = os.path.sep.join(os.path.commonprefix([obj_.filename.split(os.path.sep)
for obj_ in obj_to_extract]))
common_prefix_len = len(common_prefix)
for zipped_obj in obj_to_extract:
if zipped_obj.filename[-1:] == os.path.sep:
continue
copy_to_path(zip_file.open(zipped_obj.filename), destination_root + zipped_obj.filename[common_prefix_len:])
zip_file.close()
else:
copy_to_path(temp_file, os.path.join(destination_root, original_filename))
# remove unwanted files
clean_archive(destination_root)
# index
index_archive(doc_id, destination_root)
# prepare docset
Docset(doc).prepare()
zip_archive(doc)
return doc
def edit_doc_name(window_info: WindowInfo, doc_id: int, name: str):
UploadDoc.query(window_info).filter(pk=doc_id).update(name=name)
notify(window_info, _('Saved.'), to=WINDOW, timeout=2000)
def delete_doc_confirm(window_info: WindowInfo, doc_id: int):
doc = get_object_or_404(UploadDoc.query(window_info), pk=doc_id)
template_values = {'name': doc.name, 'doc_id': doc_id, }
html = render_to_string('updoc/delete_doc_confirm.html', template_values)
modal_show(window_info, html)
def delete_doc(request, doc_id):
obj = get_object_or_404(UploadDoc.query(request), id=doc_id)
name = obj.name
scall(request, 'updoc.delete_file', to=[SERVER], doc_id=doc_id)
messages.info(request, _('%(doc)s will be quickly deleted') % {'doc': name})
return HttpResponseRedirect(reverse('updoc:my_docs'))
def edit_doc_keywords(request: SignalRequest, doc_id: int, keywords: str):
doc = get_object_or_404(UploadDoc.query(request), pk=doc_id)
doc.keywords.clear()
for keyword in [x.strip() for x in keywords.lower().split() if x.strip()]:
if keyword:
doc.keywords.add(Keyword.get(keyword))
def edit_doc_name(request: SignalRequest, doc_id: int, name: str):
UploadDoc.query(request).filter(pk=doc_id).update(name=name)
def delete_doc_confirm(request: SignalRequest, doc_id: int):
doc = get_object_or_404(UploadDoc.query(request), pk=doc_id)
template_values = {'name': doc.name, 'doc_id': doc_id, }
html = render_to_string('updoc/delete_doc_confirm.html', template_values)
return [{'signal': 'df.modal.show', 'options': {'html': html, }, }]
