コード例 #1
0
ファイル: addpackage.py プロジェクト: d9pouces/Updoc
    def handle(self, *args, **options):
        # require
        user = None
        if options['user'] is not None:
            user = get_user_model().objects.get(username=options['user'])

        filename = options['filename']
        if filename.startswith('file://'):
            filename = filename[7:]
        if filename.startswith('http://') or filename.startswith('https://'):
            parsed_url = urlparse(filename)
            basename = os.path.basename(parsed_url.path)
            src_file = tempfile.NamedTemporaryFile()
            req = requests.get(filename, stream=True)
            chunk_size = 16384
            for chunk in req.iter_content(chunk_size):
                src_file.write(chunk)
        else:
            src_file = open(filename, mode='rb')
            basename = os.path.basename(filename)

        obj = UploadDoc(uid=str(uuid.uuid1()), user=user, name=options['name'] or basename)
        obj.save()
        process_uploaded_file(obj, src_file, basename)
        for keyword in options['keyword']:
            obj.keywords.add(Keyword.get(keyword.lower().strip()))
        obj.save()
コード例 #2
0
ファイル: signals.py プロジェクト: pombredanne/Updoc
def process_file(request: SignalRequest, doc_id: int, filename: str, original_filename: str):
    """
    * uncompress file if needed otherwise copy it
    * index its content
    * remove it

    Remove all if an error occurred

    :param request:
    :param doc_id:
    :param filename:
    :param original_filename:
    """
    temp_file = None
    destination_root = None
    doc = None
    # noinspection PyBroadException
    try:
        temp_file = open(filename, 'rb')
        doc = UploadDoc.query(request).get(pk=doc_id)
        destination_root = os.path.join(settings.MEDIA_ROOT, 'docs', doc.uid[0:2], doc.uid)
        process_uploaded_file(doc, temp_file, original_filename=original_filename, destination_root=destination_root)
        call('df.messages.info', request, sharing=SESSION, html=_('%(name)s has been uploaded and indexed') % {'name': doc.name})
    except Exception as e:
        if destination_root and os.path.isdir(destination_root):
            shutil.rmtree(destination_root)
        UploadDoc.query(request).filter(pk=doc_id).delete()
        if doc:
            call('df.messages.error', request, sharing=SESSION, html=_('An error happened during the processing of %(name)s: %(error)s') % {'name': doc.name, 'error': str(e)})
        else:
            call('df.messages.error', request, sharing=SESSION, html=_('Unable to process query'))
    finally:
        if temp_file:
            temp_file.close()
        os.remove(filename)
コード例 #3
0
ファイル: views.py プロジェクト: d9pouces/Updoc
def upload_doc_api(request):
    user = request.user if request.user.is_authenticated else None
    if user is None:
        return HttpResponse(_('You must be logged to upload files.\n'), status=401)
    elif request.method != 'POST':
        return HttpResponse(_('Only POST requests are allowed.\n'), status=400)
    form = UploadApiForm(request.GET)
    if not form.is_valid():
        return HttpResponse(_('You must supply filename, name and keywords in your query.\n'), status=400)
    tmp_file = tempfile.NamedTemporaryFile(mode='wb', dir=settings.FILE_UPLOAD_TEMP_DIR, delete=False)
    c = False
    chunk = request.read(32768)
    while chunk:
        tmp_file.write(chunk)
        c = True
        chunk = request.read(32768)
    tmp_file.flush()
    if not c:
        os.remove(tmp_file.name)
        return HttpResponse(_('Empty file. You must POST a valid file.\n'), status=400)
    # ok, we have the tmp file

    existing_objs = list(UploadDoc.query(request).filter(name=form.cleaned_data['name'])[0:1])
    if existing_objs:
        doc = existing_objs[0]
        doc.keywords.clear()
    else:
        doc = UploadDoc(uid=str(uuid.uuid1()), name=form.cleaned_data['name'], user=user)
        doc.save()
    for keyword in strip_split(form.cleaned_data['keywords'].lower()):
        doc.keywords.add(Keyword.get(keyword))
    scall(request, 'updoc.process_file', to=[SERVER], doc_id=doc.id, filename=tmp_file.name,
          original_filename=os.path.basename(form.cleaned_data['filename']))
    return HttpResponse(_('File successfully uploaded. It will be uncompressed and indexed.\n'), status=200)
コード例 #4
0
ファイル: signals.py プロジェクト: d9pouces/Updoc
def edit_doc_keywords(window_info: WindowInfo, doc_id: int, keywords: str):
    doc = get_object_or_404(UploadDoc.query(window_info), pk=doc_id)
    doc.keywords.clear()
    for keyword in [x.strip() for x in keywords.lower().split() if x.strip()]:
        if keyword:
            doc.keywords.add(Keyword.get(keyword))
    notify(window_info, _('Saved.'), to=WINDOW, timeout=2000)
コード例 #5
0
ファイル: signals.py プロジェクト: d9pouces/Updoc
def process_file(window_info: WindowInfo, doc_id: int, filename: str, original_filename: str):
    """
    * uncompress file if needed otherwise copy it
    * index its content
    * remove it

    Remove all if an error occurred

    :param window_info:
    :param doc_id:
    :param filename:
    :param original_filename:
    """
    temp_file = None
    destination_root = None
    doc = None
    # noinspection PyBroadException
    try:
        temp_file = open(filename, 'rb')
        doc = UploadDoc.query(window_info).get(pk=doc_id)
        assert isinstance(doc, UploadDoc)
        content = _('%(name)s is being processed…') % {'name': doc.name}
        notify(window_info, content=content, style=NOTIFICATION, level=INFO, timeout=5000, to=WINDOW)
        destination_root = doc.uncompressed_root
        process_uploaded_file(doc, temp_file, original_filename=original_filename)
        content = _('%(name)s has been uploaded and indexed') % {'name': doc.name}
        notify(window_info, content=content, style=NOTIFICATION, level=SUCCESS, timeout=5000, to=USER)
    except Exception as e:
        logger.exception(e)
        if destination_root and os.path.isdir(destination_root):
            shutil.rmtree(destination_root)
        UploadDoc.query(window_info).filter(pk=doc_id).delete()
        if doc:
            content = _('An error happened during the processing of %(name)s: %(error)s') % \
                      {'name': doc.name, 'error': str(e)}
            notify(window_info, content, to=[USER], style=NOTIFICATION, level=DANGER, timeout=7000)
        else:
            content = _('Unable to process query')
            notify(window_info, content, to=[USER], style=NOTIFICATION, level=DANGER, timeout=7000)
    finally:
        if temp_file:
            temp_file.close()
        os.remove(filename)
コード例 #6
0
ファイル: views.py プロジェクト: d9pouces/Updoc
def upload(request):
    """Index view, displaying and processing a form."""
    set_websocket_topics(request)
    if request.method == 'POST':
        form = MetadatadUploadForm(request.POST)
        if form.is_valid():
            messages.info(request, _('File successfully uploaded'))
            obj = get_object_or_404(UploadDoc.query(request), id=form.cleaned_data['pk'])
            obj.name = form.cleaned_data['name']
            for keyword in form.cleaned_data['keywords'].lower().split():
                obj.keywords.add(Keyword.get(keyword))
            obj.save()
            return HttpResponseRedirect(reverse('upload'))
        elif 'pk' in form.cleaned_data:
            obj = get_object_or_404(UploadDoc.query(request), id=form.cleaned_data['pk'])
            obj.delete()
            messages.error(request, _('Unable to upload this file'))
        else:
            messages.error(request, _('Unable to upload this file'))
    else:
        form = FileUploadForm()
    template_values = {'form': form, 'title': _('Upload a new file'), 'root_host': settings.SERVER_BASE_URL[:-1]}
    return TemplateResponse(request, 'updoc/upload.html', template_values)
コード例 #7
0
ファイル: views.py プロジェクト: d9pouces/Updoc
def upload_doc_progress(request):
    form = FileUploadForm(request.POST, request.FILES)
    if not form.is_valid():
        raise PermissionDenied
    uploaded_file = request.FILES['file']
    tmp_file = tempfile.NamedTemporaryFile(mode='wb', dir=settings.FILE_UPLOAD_TEMP_DIR, delete=False)
    chunk = uploaded_file.read(16384)
    while chunk:
        tmp_file.write(chunk)
        chunk = uploaded_file.read(16384)
    tmp_file.flush()

    basename = os.path.basename(uploaded_file.name).rpartition('.')[0]
    if basename.endswith('.tar'):
        basename = basename[:-4]
    doc = UploadDoc(name=basename, user=request.user if request.user.is_authenticated else None,
                    uid=str(uuid.uuid1()))
    doc.save()
    scall(request, 'updoc.process_file', to=[SERVER], doc_id=doc.id, filename=tmp_file.name,
          original_filename=uploaded_file.name)
    # offer a correct name for the newly uploaded document
    form = MetadatadUploadForm(initial={'pk': doc.pk, 'name': basename, })
    template_values = {'form': form, }
    return TemplateResponse(request, 'updoc/upload_doc_progress.html', template_values)
コード例 #8
0
ファイル: views.py プロジェクト: d9pouces/Updoc
def my_docs(request):
    if request.user.is_authenticated:
        set_websocket_topics(request)
    user = request.user if request.user.is_authenticated else None
    if request.method == 'POST' and user and user.has_perm('updoc.add_uploaddoc'):
        form = UrlRewriteForm(request.POST)
        if form.is_valid():
            RewrittenUrl(user=user, src=form.cleaned_data['src'], dst=form.cleaned_data['dst']).save()
            messages.info(request, _('URL %(src)s will be rewritten as %(dst)s') % form.cleaned_data)
            return HttpResponseRedirect(reverse('updoc:my_docs'))
    else:
        form = UrlRewriteForm()
    uploads = UploadDoc.query(request).order_by('-upload_time').select_related()
    rw_urls = RewrittenUrl.query(request).order_by('src')
    template_values = {'uploads': uploads, 'title': _('My documents'), 'rw_urls': rw_urls,
                       'rw_form': form, 'editable': True, 'has_search_results': False, }
    return TemplateResponse(request, 'updoc/my_docs.html', template_values)
コード例 #9
0
ファイル: process.py プロジェクト: d9pouces/Updoc
def process_uploaded_file(doc: UploadDoc, temp_file, original_filename: str):
    """
    * Clean previous content if needed
    * Uncompress content
    * Index content

    :param doc:
    :param temp_file: file descriptor
    :param original_filename: nom d'origine du fichier
    """
    assert isinstance(doc, UploadDoc)
    destination_root = doc.uncompressed_root
    doc_id = doc.id
    doc_uid = doc.uid
    UploadDoc.objects.filter(id=doc_id).update(path=destination_root, version=F('version') + 1)
    doc.path = destination_root
    doc.version += 1
    # first, we clean previous index and data
    doc.clean_archive()

    # ok, let's go to decompress
    destination_root += os.path.sep
    if original_filename[-7:] in ('.tar.gz', '.tar.xz') or original_filename[-8:] == '.tar.bz2'\
            or original_filename[-4:] in ('.tar', '.tbz', '.tgz', '.txz'):
        tar_file = tarfile.open(name=original_filename, mode='r:*', fileobj=temp_file)
        names = [name_ for name_ in tar_file.getnames() if os.path.join(doc_uid, name_).startswith(doc_uid)]
        common_prefix = os.path.sep.join(os.path.commonprefix([name_.split(os.path.sep) for name_ in names]))
        common_prefix_len = len(common_prefix)
        for member in tar_file.getmembers():
            if not os.path.join(doc_uid, member.name).startswith(doc_uid):
                continue
            dst_path = destination_root + member.name[common_prefix_len:]
            if member.isdir():
                os.makedirs(dst_path, mode=0o777, exist_ok=True)
            elif member.issym():
                if not os.path.join(doc_uid, member.linkname).startswith(doc_uid):
                    continue
                os.symlink(destination_root + member.linkname[common_prefix_len:], dst_path)
            elif member.isfile():
                copy_to_path(tar_file.extractfile(member), dst_path)
        tar_file.close()
    elif original_filename[-4:] == '.zip':
        zip_file = zipfile.ZipFile(temp_file, mode='r')

        obj_to_extract = [zipped_obj for zipped_obj in zip_file.infolist()
                          if os.path.join(doc_uid, zipped_obj.filename).startswith(doc_uid)]
        common_prefix = os.path.sep.join(os.path.commonprefix([obj_.filename.split(os.path.sep)
                                                               for obj_ in obj_to_extract]))
        common_prefix_len = len(common_prefix)
        for zipped_obj in obj_to_extract:
            if zipped_obj.filename[-1:] == os.path.sep:
                continue
            copy_to_path(zip_file.open(zipped_obj.filename), destination_root + zipped_obj.filename[common_prefix_len:])
        zip_file.close()
    else:
        copy_to_path(temp_file, os.path.join(destination_root, original_filename))
    # remove unwanted files
    clean_archive(destination_root)
    # index
    index_archive(doc_id, destination_root)
    # prepare docset
    Docset(doc).prepare()
    zip_archive(doc)
    return doc
コード例 #10
0
ファイル: signals.py プロジェクト: d9pouces/Updoc
def edit_doc_name(window_info: WindowInfo, doc_id: int, name: str):
    UploadDoc.query(window_info).filter(pk=doc_id).update(name=name)
    notify(window_info, _('Saved.'), to=WINDOW, timeout=2000)
コード例 #11
0
ファイル: signals.py プロジェクト: d9pouces/Updoc
def delete_doc_confirm(window_info: WindowInfo, doc_id: int):
    doc = get_object_or_404(UploadDoc.query(window_info), pk=doc_id)
    template_values = {'name': doc.name, 'doc_id': doc_id, }
    html = render_to_string('updoc/delete_doc_confirm.html', template_values)
    modal_show(window_info, html)
コード例 #12
0
ファイル: views.py プロジェクト: d9pouces/Updoc
def delete_doc(request, doc_id):
    obj = get_object_or_404(UploadDoc.query(request), id=doc_id)
    name = obj.name
    scall(request, 'updoc.delete_file', to=[SERVER], doc_id=doc_id)
    messages.info(request, _('%(doc)s will be quickly deleted') % {'doc': name})
    return HttpResponseRedirect(reverse('updoc:my_docs'))
コード例 #13
0
ファイル: signals.py プロジェクト: pombredanne/Updoc
def edit_doc_keywords(request: SignalRequest, doc_id: int, keywords: str):
    doc = get_object_or_404(UploadDoc.query(request), pk=doc_id)
    doc.keywords.clear()
    for keyword in [x.strip() for x in keywords.lower().split() if x.strip()]:
        if keyword:
            doc.keywords.add(Keyword.get(keyword))
コード例 #14
0
ファイル: signals.py プロジェクト: pombredanne/Updoc
def edit_doc_name(request: SignalRequest, doc_id: int, name: str):
    UploadDoc.query(request).filter(pk=doc_id).update(name=name)
コード例 #15
0
ファイル: signals.py プロジェクト: pombredanne/Updoc
def delete_doc_confirm(request: SignalRequest, doc_id: int):
    doc = get_object_or_404(UploadDoc.query(request), pk=doc_id)
    template_values = {'name': doc.name, 'doc_id': doc_id, }
    html = render_to_string('updoc/delete_doc_confirm.html', template_values)
    return [{'signal': 'df.modal.show', 'options': {'html': html, }, }]