def get_file(request):
    contract_id = request.validated['contract_id']
    document = request.validated['document']
    key = request.params.get('download')
    conn = getattr(request.registry, 's3_connection', None)
    filename = "{}_{}".format(document.id, key)
    if conn and filename not in request.validated['contract']['_attachments']:
        filename = "{}/{}/{}".format(contract_id, document.id, key)
        url = conn.generate_url(method='GET',
                                bucket=request.registry.bucket_name,
                                key=filename, expires_in=300)
        request.response.content_type = document.format.encode('utf-8')
        request.response.content_disposition = build_header(
            document.title, filename_compat=quote(document.title.encode('utf-8')))
        request.response.status = '302 Moved Temporarily'
        request.response.location = url
        return url
    else:
        filename = "{}_{}".format(document.id, key)
        data = request.registry.db.get_attachment(contract_id, filename)
        if data:
            request.response.content_type = document.format.encode('utf-8')
            request.response.content_disposition = build_header(
                document.title, filename_compat=quote(document.title.encode('utf-8')))
            request.response.body_file = data
            return request.response
        request.errors.add('url', 'download', 'Not Found')
        request.errors.status = 404
def get_file(request):
    plan_id = request.validated["plan_id"]
    document = request.validated["document"]
    key = request.params.get("download")
    conn = getattr(request.registry, "s3_connection", None)
    filename = "{}_{}".format(document.id, key)
    if conn and filename not in request.validated["plan"]["_attachments"]:
        filename = "{}/{}/{}".format(plan_id, document.id, key)
        url = conn.generate_url(method="GET", bucket=request.registry.bucket_name, key=filename, expires_in=300)
        request.response.content_type = document.format.encode("utf-8")
        request.response.content_disposition = build_header(
            document.title, filename_compat=quote(document.title.encode("utf-8"))
        )
        request.response.status = "302 Moved Temporarily"
        request.response.location = url
        return url
    else:
        filename = "{}_{}".format(document.id, key)
        data = request.registry.db.get_attachment(plan_id, filename)
        if data:
            request.response.content_type = document.format.encode("utf-8")
            request.response.content_disposition = build_header(
                document.title, filename_compat=quote(document.title.encode("utf-8"))
            )
            request.response.body_file = data
            return request.response
        request.errors.add("url", "download", "Not Found")
        request.errors.status = 404
Example #3
0
def get_file(request):
    db_doc_id = request.validated['db_doc'].id
    document = request.validated['document']
    key = request.params.get('download')
    if not any([key in i.url for i in request.validated['documents']]):
        request.errors.add('url', 'download', 'Not Found')
        request.errors.status = 404
        return
    filename = "{}_{}".format(document.id, key)
    if request.registry.docservice_url and filename not in request.validated['db_doc']['_attachments']:
        document = [i for i in request.validated['documents'] if key in i.url][-1]
        if 'Signature=' in document.url and 'KeyID' in document.url:
            url = document.url
        else:
            if 'download=' not in document.url:
                key = urlparse(document.url).path.replace('/get/', '')
            if not document.hash:
                url = generate_docservice_url(request, key, prefix='{}/{}'.format(db_doc_id, document.id))
            else:
                url = generate_docservice_url(request, key)
        request.response.content_type = document.format.encode('utf-8')
        request.response.content_disposition = build_header(document.title, filename_compat=quote(document.title.encode('utf-8')))
        request.response.status = '302 Moved Temporarily'
        request.response.location = url
        return url
    else:
        data = request.registry.db.get_attachment(db_doc_id, filename)
        if data:
            request.response.content_type = document.format.encode('utf-8')
            request.response.content_disposition = build_header(document.title, filename_compat=quote(document.title.encode('utf-8')))
            request.response.body_file = data
            return request.response
        request.errors.add('url', 'download', 'Not Found')
        request.errors.status = 404
 def upload(self, post_file, uuid=None):
     filename = get_filename(post_file.filename)
     content_type = post_file.type
     in_file = post_file.file
     bucket = self.connection.get_bucket(self.bucket)
     if uuid is None:
         uuid = uuid4().hex
         path = '/'.join([format(i, 'x') for i in UUID(uuid).fields])
         key = bucket.new_key(path)
     else:
         try:
             path = '/'.join([format(i, 'x') for i in UUID(uuid).fields])
         except ValueError:
             raise KeyNotFound(uuid)
         if path not in bucket:
             raise KeyNotFound(uuid)
         key = bucket.get_key(path)
         if key.size != 0:
             raise ContentUploaded(uuid)
         md5 = key.get_metadata('hash')
         if key.compute_md5(in_file)[0] != md5[4:]:
             raise HashInvalid(md5)
     key.set_metadata('Content-Type', content_type)
     key.set_metadata("Content-Disposition", build_header(filename, filename_compat=quote(filename.encode('utf-8'))))
     key.set_contents_from_file(in_file)
     key.set_acl('private')
     return uuid, 'md5:' + key.etag[1:-1], content_type, filename
def create_mail_attachment(attdef, payload=None):
    """Create the MIME part corresponding to the given attachment.

    Mandatory keys: 'fname', 'tmpname', 'content-type'

    :param attdef: a dictionary containing the attachment definition
    :return: a MIMEBase object
    """
    from email import Encoders
    from email.mime.base import MIMEBase

    if "content-type" in attdef:
        maintype, subtype = attdef["content-type"].split("/")
    elif "Content-Type" in attdef:
        maintype, subtype = attdef["Content-Type"].split("/")
    else:
        return None
    res = MIMEBase(maintype, subtype)
    if payload is None:
        with open(os.path.join(
                settings.MEDIA_ROOT, "webmail", attdef["tmpname"]), "rb") as fp:
            res.set_payload(fp.read())
    else:
        res.set_payload(payload)
    Encoders.encode_base64(res)
    if isinstance(attdef['fname'], str):
        attdef['fname'] = attdef['fname'].decode('utf-8')
    res['Content-Disposition'] = build_header(attdef['fname'])
    return res
Example #6
0
def getattachment(request):
    """Fetch a message attachment

    FIXME: par manque de caching, le bodystructure du message est
    redemandé pour accéder aux headers de cette pièce jointe.

    :param request: a ``Request`` object
    """
    mbox = request.GET.get("mbox", None)
    mailid = request.GET.get("mailid", None)
    pnum = request.GET.get("partnumber", None)
    if not mbox or not mailid or not pnum:
        raise WebmailError(_("Invalid request"))

    imapc = get_imapconnector(request)
    partdef, payload = imapc.fetchpart(mailid, mbox, pnum)
    resp = HttpResponse(decode_payload(partdef["encoding"], payload))
    resp["Content-Type"] = partdef["Content-Type"]
    resp["Content-Transfer-Encoding"] = partdef["encoding"]
    if partdef["disposition"] != 'NIL':
        disp = partdef["disposition"]
        # FIXME : ugly hack, see fetch_parser.py for more explanation
        # :p
        if type(disp[1][0]) != dict:
            cd = '%s; %s=%s' % (disp[0], disp[1][0], disp[1][1])
        else:
            cd = '%s; %s=%s' % (disp[0], disp[1][0]['struct'][0], disp[1][0]['struct'][1])
    else:
        cd = build_header(request.GET["fname"])
    resp["Content-Disposition"] = cd
    resp["Content-Length"] = partdef["size"]
    return resp
Example #7
0
    def image(self, subpath=None):
        """Return the image in a specific scale, either inline
        (default) or as attachment.

        :param subpath: [<image_scale>]/download] (optional).
                        When 'download' is the last element in subpath,
                        the image is served with a 'Content-Disposition:
                        attachment' header.  <image_scale> has to be one of the
                        predefined image_scales - either from the defaults in
                        this module or one set with a
                        kotti.image_scales.<scale_name> in your app config ini
                        file.
        :type subpath: str

        :result: complete response object
        :rtype: pyramid.response.Response
        """

        if subpath is None:
            subpath = self.request.subpath

        width, height = (None, None)
        subpath = list(subpath)

        if (len(subpath) > 0) and (subpath[-1] == "download"):
            disposition = "attachment"
            subpath.pop()
        else:
            disposition = "inline"

        if len(subpath) == 1:
            scale = subpath[0]
            if scale in image_scales:
                # /path/to/image/scale/thumb
                width, height = image_scales[scale]

        if not (width and height):
            return self.request.uploaded_file_response(
                self.context.data, disposition)

        image, format, size = scaleImage(self.context.data.file.read(),
                                         width=width,
                                         height=height,
                                         direction="thumb")
        res = Response(
            headerlist=[
                ('Content-Disposition', '{0};filename="{1}"'.format(
                    disposition,
                    self.context.filename.encode('ascii', 'ignore'))),
                ('Content-Length', str(len(image))),
                ('Content-Type', str(self.context.mimetype)),
            ],
            body=image,
        )
        res.content_disposition = rfc6266.build_header(
            self.context.filename, disposition=disposition,
            filename_compat=unidecode(self.context.filename))

        return res
Example #8
0
    def __init__(self, f, request, disposition='attachment',
                 cache_max_age=604800, content_type=None,
                 content_encoding=None):
        """
        :param f: the ``UploadedFile`` file field value.
        :type f: :class:`depot.io.interfaces.StoredFile`

        :param request: Current request.
        :type request: :class:`pyramid.request.Request`

        :param disposition:
        :type disposition:

        :param cache_max_age: The number of seconds that should be used to HTTP
                              cache this response.

        :param content_type: The content_type of the response.

        :param content_encoding: The content_encoding of the response.
                                 It's generally safe to leave this set to
                                 ``None`` if you're serving a binary file.
                                 This argument will be ignored if you also
                                 leave ``content-type`` as ``None``.
        """

        if f.public_url:
            raise HTTPMovedPermanently(f.public_url)

        content_encoding, content_type = self._get_type_and_encoding(
            content_encoding, content_type, f)

        super(StoredFileResponse, self).__init__(
            conditional_response=True,
            content_type=content_type,
            content_encoding=content_encoding)

        app_iter = None
        if request is not None and \
                not get_settings()['kotti.depot_replace_wsgi_file_wrapper']:
            environ = request.environ
            if 'wsgi.file_wrapper' in environ:
                app_iter = environ['wsgi.file_wrapper'](f, _BLOCK_SIZE)
        if app_iter is None:
            app_iter = FileIter(f)
        self.app_iter = app_iter

        # assignment of content_length must come after assignment of app_iter
        self.content_length = f.content_length
        self.last_modified = f.last_modified

        if cache_max_age is not None:
            self.cache_expires = cache_max_age
            self.cache_control.public = True

        self.etag = self.generate_etag(f)
        self.content_disposition = rfc6266.build_header(
            f.filename, disposition=disposition,
            filename_compat=unidecode(f.filename))
def upload_file(request, blacklisted_fields=DOCUMENT_BLACKLISTED_FIELDS):
    first_document = (
        request.validated["documents"][0]
        if "documents" in request.validated and request.validated["documents"]
        else None
    )
    if request.content_type == "multipart/form-data":
        data = request.validated["file"]
        filename = get_filename(data)
        content_type = data.type
        in_file = data.file
    else:
        filename = first_document.title
        content_type = request.content_type
        in_file = request.body_file

    if hasattr(request.context, "documents"):
        # upload new document
        model = type(request.context).documents.model_class
    else:
        # update document
        model = type(request.context)
    document = model({"title": filename, "format": content_type})
    document.__parent__ = request.context
    if "document_id" in request.validated:
        document.id = request.validated["document_id"]
    if first_document:
        for attr_name in type(first_document)._fields:
            if attr_name not in blacklisted_fields:
                setattr(document, attr_name, getattr(first_document, attr_name))
    key = generate_id()
    document_route = request.matched_route.name.replace("collection_", "")
    document_path = request.current_route_path(
        _route_name=document_route, document_id=document.id, _query={"download": key}
    )
    document.url = "/" + "/".join(document_path.split("/")[3:])
    conn = getattr(request.registry, "s3_connection", None)
    if conn:
        bucket = conn.get_bucket(request.registry.bucket_name)
        filename = "{}/{}/{}".format(request.validated["plan_id"], document.id, key)
        key = bucket.new_key(filename)
        key.set_metadata("Content-Type", document.format)
        key.set_metadata(
            "Content-Disposition", build_header(document.title, filename_compat=quote(document.title.encode("utf-8")))
        )
        key.set_contents_from_file(in_file)
        key.set_acl("private")
    else:
        filename = "{}_{}".format(document.id, key)
        request.validated["plan"]["_attachments"][filename] = {
            "content_type": document.format,
            "data": b64encode(in_file.read()),
        }
    update_logging_context(request, {"file_size": in_file.tell()})
    return document
Example #10
0
def _export(content, filename):
    """Export a csv file's content

    :param content: the content to export (string)
    :param filename: the name that will appear into the response
    :return: an ``HttpResponse`` object
    """
    resp = HttpResponse(content)
    resp["Content-Type"] = "text/csv"
    resp["Content-Length"] = len(content)
    resp["Content-Disposition"] = build_header(filename)
    return resp
Example #11
0
def download_filters_set(request, name):
    sc = SieveClient(user=request.user.username,
                     password=request.session["password"])
    try:
        script = sc.getscript(name)
    except SieveClientError as e:
        return ajax_response(request, "ko", respmsg=str(e))

    resp = HttpResponse(script)
    resp["Content-Type"] = "text/plain; charset=utf-8"
    resp["Content-Length"] = len(script)
    resp["Content-Disposition"] = build_header('%s.txt' % name)
    return resp
def upload_file(request, blacklisted_fields=DOCUMENT_BLACKLISTED_FIELDS):
    first_document = request.validated['documents'][0] if 'documents' in request.validated and request.validated['documents'] else None
    if request.content_type == 'multipart/form-data':
        data = request.validated['file']
        filename = get_filename(data)
        content_type = data.type
        in_file = data.file
    else:
        filename = first_document.title
        content_type = request.content_type
        in_file = request.body_file

    if hasattr(request.context, "documents"):
        # upload new document
        model = type(request.context).documents.model_class
    else:
        # update document
        model = type(request.context)
    document = model({'title': filename, 'format': content_type})
    document.__parent__ = request.context
    if 'document_id' in request.validated:
        document.id = request.validated['document_id']
    if first_document:
        for attr_name in type(first_document)._fields:
            if attr_name not in blacklisted_fields:
                setattr(document, attr_name, getattr(first_document, attr_name))
    key = generate_id()
    document_route = request.matched_route.name.replace("collection_", "")
    document_path = request.current_route_path(_route_name=document_route, document_id=document.id, _query={'download': key})
    document.url = '/' + '/'.join(document_path.split('/')[3:])
    conn = getattr(request.registry, 's3_connection', None)
    if conn:
        bucket = conn.get_bucket(request.registry.bucket_name)
        filename = "{}/{}/{}".format(request.validated['contract_id'],
                                     document.id, key)
        key = bucket.new_key(filename)
        key.set_metadata('Content-Type', document.format)
        key.set_metadata("Content-Disposition", build_header(document.title, filename_compat=quote(document.title.encode('utf-8'))))
        key.set_contents_from_file(in_file)
        key.set_acl('private')
    else:
        filename = "{}_{}".format(document.id, key)
        request.validated['contract']['_attachments'][filename] = {
            "content_type": document.format,
            "data": b64encode(in_file.read())
        }
    update_logging_context(request, {'file_size': in_file.tell()})
    return document
Example #13
0
def get_account_credentials(request, accountid):
    """View to download a document."""
    account = User.objects.get(pk=accountid)
    if not request.user.can_access(account):
        raise PermDeniedException()
    fname = get_creds_filename(account)
    if not os.path.exists(fname):
        raise ModoboaException(_("No document available for this user"))
    content = decrypt_file(fname)
    if param_tools.get_global_parameter("delete_first_dl"):
        os.remove(fname)
    resp = HttpResponse(content)
    resp["Content-Type"] = "application/pdf"
    resp["Content-Length"] = len(content)
    resp["Content-Disposition"] = build_header(os.path.basename(fname))
    return resp
Example #14
0
    def zip_selected(self, objects):
        response = self.request.response
        settings = getUtility(IRegistry).forInterface(IZipExportSettings)

        with ZipGenerator() as generator:

            for obj in objects:
                repre = getMultiAdapter((obj, self.request),
                                        interface=IZipRepresentation)

                for path, pointer in repre.get_files():
                    if not pointer:
                        if settings.include_empty_folders:
                            generator.add_folder(path)
                        continue

                    try:
                        generator.add_file(path, pointer)
                    except LargeZipFile:
                        messages = IStatusMessage(self.request)
                        messages.add(_("statmsg_zip_file_too_big",
                                       default=u"Content is too big "
                                       "to export"),
                                     type=u"error")
                        return self.request.response.redirect(
                            self.context.absolute_url())

            # check if zip has files
            if generator.is_empty:
                raise NoExportableContent()

            zip_file = generator.generate()

            # Trigger the per container event
            notify(ContainerZippedEvent(self.context))

            # Generate response file
            filename = u'%s.zip' % self.context.title
            response.setHeader(
                "Content-Disposition",
                build_header(filename, disposition='attachment'))
            response.setHeader("Content-type", "application/zip")
            response.setHeader("Content-Length",
                               os.stat(zip_file.name).st_size)

            return filestream_iterator(zip_file.name, 'rb')
Example #15
0
def create_mail_attachment(attdef):
    """Create the MIME part corresponding to the given attachment.

    Mandatory keys: 'fname', 'tmpname', 'content-type'

    :param attdef: a dictionary containing the attachment definition
    :return: a MIMEBase object
    """
    from email import Encoders
    from email.mime.base import MIMEBase

    maintype, subtype = attdef["content-type"].split("/")
    res = MIMEBase(maintype, subtype)
    fp = open(os.path.join(settings.MEDIA_ROOT, "webmail", attdef["tmpname"]), "rb")
    res.set_payload(fp.read())
    fp.close()
    Encoders.encode_base64(res)
    res['Content-Disposition'] = build_header(attdef['fname'].decode('utf-8'))
    return res
def upload_file(request):
    first_document = request.validated['documents'][0] if 'documents' in request.validated and request.validated['documents'] else None
    if request.content_type == 'multipart/form-data':
        data = request.validated['file']
        filename = get_filename(data)
        content_type = data.type
        in_file = data.file
    else:
        filename = first_document.title
        content_type = request.content_type
        in_file = request.body_file
    document = Document({
        'title': filename,
        'format': content_type
    })
    document.__parent__ = request.context
    if 'document_id' in request.validated:
        document.id = request.validated['document_id']
    if first_document:
        document.datePublished = first_document.datePublished
    key = generate_id()
    document_route = request.matched_route.name.replace("collection_", "")
    document_path = request.current_route_path(_route_name=document_route, document_id=document.id, _query={'download': key})
    document.url = '/tenders' + document_path.split('/tenders', 1)[1]
    conn = getattr(request.registry, 's3_connection', None)
    if conn:
        bucket = conn.get_bucket(request.registry.bucket_name)
        filename = "{}/{}/{}".format(request.validated['tender_id'], document.id, key)
        key = bucket.new_key(filename)
        key.set_metadata('Content-Type', document.format)
        key.set_metadata("Content-Disposition", build_header(document.title, filename_compat=quote(document.title.encode('utf-8'))))
        key.set_contents_from_file(in_file)
        key.set_acl('private')
    else:
        filename = "{}_{}".format(document.id, key)
        request.validated['tender']['_attachments'][filename] = {
            "content_type": document.format,
            "data": b64encode(in_file.read())
        }
    return document
 def upload(self, post_file, uuid=None):
     filename = get_filename(post_file.filename)
     content_type = post_file.type
     in_file = post_file.file
     if uuid is not None:
         if uuid not in self.storage:
             raise KeyNotFound(uuid)
         if self.storage[uuid]['Content']:
             raise ContentUploaded(uuid)
         key = self.storage[uuid]
     else:
         uuid = uuid4().hex
         key = self.storage[uuid] = {}
     content = in_file.read()
     key_md5 = key.get('hash')
     md5hash = 'md5:' + md5(content).hexdigest()
     if key_md5 and md5hash != key_md5:
         raise HashInvalid(key_md5)
     key['hash'] = md5hash
     key['Content-Type'] = content_type
     key["Content-Disposition"] = build_header(filename, filename_compat=quote(filename.encode('utf-8')))
     key['Content'] = content
     return uuid, md5hash, content_type, filename
Example #18
0
def getattachment(request):
    """Fetch a message attachment

    FIXME: par manque de caching, le bodystructure du message est
    redemandé pour accéder aux headers de cette pièce jointe.

    :param request: a ``Request`` object
    """
    mbox = request.GET.get("mbox", None)
    mailid = request.GET.get("mailid", None)
    pnum = request.GET.get("partnumber", None)
    fname = request.GET.get("fname", None)
    if not mbox or not mailid or not pnum or not fname:
        raise BadRequest(_("Invalid request"))

    imapc = get_imapconnector(request)
    partdef, payload = imapc.fetchpart(mailid, mbox, pnum)
    resp = HttpResponse(decode_payload(partdef["encoding"], payload))
    resp["Content-Type"] = partdef["Content-Type"]
    resp["Content-Transfer-Encoding"] = partdef["encoding"]
    resp["Content-Disposition"] = build_header(fname)
    if int(partdef["size"]) < 200:
        resp["Content-Length"] = partdef["size"]
    return resp
Example #19
0
 def upload(self, post_file, uuid=None):
     filename = get_filename(post_file.filename)
     content_type = post_file.type
     in_file = post_file.file
     if uuid is not None:
         if uuid not in self.storage:
             raise KeyNotFound(uuid)
         if self.storage[uuid]['Content']:
             raise ContentUploaded(uuid)
         key = self.storage[uuid]
     else:
         uuid = uuid4().hex
         key = self.storage[uuid] = {}
     content = in_file.read()
     key_md5 = key.get('hash')
     md5hash = 'md5:' + md5(content).hexdigest()
     if key_md5 and md5hash != key_md5:
         raise HashInvalid(key_md5)
     key['hash'] = md5hash
     key['Content-Type'] = content_type
     key["Content-Disposition"] = build_header(
         filename, filename_compat=quote(filename.encode('utf-8')))
     key['Content'] = content
     return uuid, md5hash, content_type, filename
Example #20
0
    def encode_hdr(self, boundary):
        """Returns the header of the encoding of this parameter"""
        if not self._encoded_hdr or self._encoded_bdr != boundary:
            boundary = self.quote(boundary)
            self._encoded_bdr = boundary
            headers = ["--%s" % boundary]
            if self.fname:
                disposition = build_header(self.fname,
                                           disposition='form-data',
                                           name=self.name).encode()
            else:
                disposition = 'form-data; name="%s"' % self.name

            headers.append("Content-Disposition: %s" % disposition)
            if self.filetype:
                filetype = self.filetype
            else:
                filetype = "text/plain; charset=utf-8"
            headers.append("Content-Type: %s" % filetype)
            headers.append("Content-Length: %i" % self.size)
            headers.append("")
            headers.append("")
            self._encoded_hdr = CRLF.join(headers)
        return self._encoded_hdr
Example #21
0
def getattachment(request):
    """Fetch a message attachment

    FIXME: par manque de caching, le bodystructure du message est
    redemandé pour accéder aux headers de cette pièce jointe.

    :param request: a ``Request`` object
    """
    mbox = request.GET.get("mbox", None)
    mailid = request.GET.get("mailid", None)
    pnum = request.GET.get("partnumber", None)
    fname = request.GET.get("fname", None)
    if not mbox or not mailid or not pnum or not fname:
        raise BadRequest(_("Invalid request"))

    imapc = get_imapconnector(request)
    partdef, payload = imapc.fetchpart(mailid, mbox, pnum)
    resp = HttpResponse(decode_payload(partdef["encoding"], payload))
    resp["Content-Type"] = partdef["Content-Type"]
    resp["Content-Transfer-Encoding"] = partdef["encoding"]
    resp["Content-Disposition"] = build_header(fname)
    if int(partdef["size"]) < 200:
        resp["Content-Length"] = partdef["size"]
    return resp
Example #22
0
 def make_content_disposition_header(fn):
     return build_header(os.path.basename(fn)).decode('ascii')
def video_encodings_download(request, course_key_string):
    """
    Returns a CSV report containing the encoded video URLs for video uploads
    in the following format:

    Video ID,Name,Status,Profile1 URL,Profile2 URL
    aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4
    """
    course = _get_and_validate_course(course_key_string, request.user)

    if not course:
        return HttpResponseNotFound()

    def get_profile_header(profile):
        """Returns the column header string for the given profile's URLs"""
        # Translators: This is the header for a CSV file column
        # containing URLs for video encodings for the named profile
        # (e.g. desktop, mobile high quality, mobile low quality)
        return _("{profile_name} URL").format(profile_name=profile)

    profile_whitelist = VideoUploadConfig.get_profile_whitelist()

    videos = list(_get_videos(course))
    name_col = _("Name")
    duration_col = _("Duration")
    added_col = _("Date Added")
    video_id_col = _("Video ID")
    status_col = _("Status")
    profile_cols = [get_profile_header(profile) for profile in profile_whitelist]

    def make_csv_dict(video):
        """
        Makes a dictionary suitable for writing CSV output. This involves
        extracting the required items from the original video dict and
        converting all keys and values to UTF-8 encoded string objects,
        because the CSV module doesn't play well with unicode objects.
        """
        # Translators: This is listed as the duration for a video that has not
        # yet reached the point in its processing by the servers where its
        # duration is determined.
        duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending")
        ret = dict(
            [
                (name_col, video["client_video_id"]),
                (duration_col, duration_val),
                (added_col, video["created"].isoformat()),
                (video_id_col, video["edx_video_id"]),
                (status_col, video["status"]),
            ] +
            [
                (get_profile_header(encoded_video["profile"]), encoded_video["url"])
                for encoded_video in video["encoded_videos"]
                if encoded_video["profile"] in profile_whitelist
            ]
        )
        return {
            key.encode("utf-8"): value.encode("utf-8")
            for key, value in ret.items()
        }

    response = HttpResponse(content_type="text/csv")
    # Translators: This is the suggested filename when downloading the URL
    # listing for videos uploaded through Studio
    filename = _("{course}_video_urls").format(course=course.id.course)
    # See https://tools.ietf.org/html/rfc6266#appendix-D
    response["Content-Disposition"] = rfc6266.build_header(
        filename + ".csv",
        filename_compat="video_urls.csv"
    )
    writer = csv.DictWriter(
        response,
        [
            col_name.encode("utf-8")
            for col_name
            in [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols
        ],
        dialect=csv.excel
    )
    writer.writeheader()
    for video in videos:
        writer.writerow(make_csv_dict(video))
    return response
Example #24
0
    def serve_raw_file(self,
                       file,
                       content_type=None,
                       disposition=None,
                       name=None):
        # Adapted from CherryPy's serve_file(), modified to work with file-like
        # objects

        response = cherrypy.response
        st = None

        if isinstance(file, str):

            path = file

            if not os.path.isabs(path):
                raise ValueError("'%s' is not an absolute path." % path)

            try:
                st = os.stat(path)
            except OSError:
                raise cherrypy.NotFound()

            if stat.S_ISDIR(st.st_mode):
                raise cherrypy.NotFound()

            response.headers['Last-Modified'] = httputil.HTTPDate(st.st_mtime)
            cptools.validate_since()
            file = open(path, "rb")
        else:
            path = getattr(file, "name", None)
            if path:
                try:
                    st = os.stat(path)
                except OSError:
                    pass
            else:
                if not hasattr(file, "read"):
                    raise ValueError(
                        "Expected a file-like object, got %r instead "
                        "(object has no read() method)" % file)
                if not hasattr(file, "seek"):
                    raise ValueError("Can't serve file-like object %r "
                                     "(object has no seek() method)" % file)
                if not hasattr(file, "tell"):
                    raise ValueError("Can't serve file-like object %r "
                                     "(object has no tell() method)" % file)

        # Set the content type
        if content_type is None:

            if path:
                content_type = mimetypes.guess_type(path)[0]

            if not content_type:
                content_type = "text/plain"

        response.headers["Content-Type"] = content_type

        # Set the content disposition
        if disposition is not None:
            cd = disposition
            if not name and path:
                name = os.path.basename(path)
            if name:
                cd = rfc6266.build_header(name, cd)
            response.headers["Content-Disposition"] = cd

        if self.use_xsendfile and path:
            response.headers["X-Sendfile"] = path
            return ""

        # Find the size of the file
        if st is None:
            start = file.tell()
            file.seek(0, 2)  # Move to the end of the file
            c_len = file.tell() - start
            file.seek(start)
        else:
            c_len = st.st_size

        # HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code
        if cherrypy.request.protocol >= (1, 1):
            response.headers["Accept-Ranges"] = "bytes"
            r = httputil.get_ranges(cherrypy.request.headers.get('Range'),
                                    c_len)
            if r == []:
                response.headers['Content-Range'] = "bytes */%s" % c_len
                message = "Invalid Range (first-byte-pos greater than Content-Length)"
                raise cherrypy.HTTPError(416, message)
            if r:
                if len(r) == 1:
                    # Return a single-part response.
                    start, stop = r[0]
                    if stop > c_len:
                        stop = c_len
                    r_len = stop - start
                    response.status = "206 Partial Content"
                    response.headers['Content-Range'] = (
                        "bytes %s-%s/%s" % (start, stop - 1, c_len))
                    response.headers['Content-Length'] = r_len
                    file.seek(start)
                    response.body = file_generator_limited(file, r_len)
                else:
                    # Return a multipart/byteranges response.
                    response.status = "206 Partial Content"
                    import mimetools
                    boundary = mimetools.choose_boundary()
                    ct = "multipart/byteranges; boundary=%s" % boundary
                    response.headers['Content-Type'] = ct
                    if "Content-Length" in response.headers:
                        # Delete Content-Length header so finalize() recalcs it.
                        del response.headers["Content-Length"]

                    def file_ranges():
                        # Apache compatibility:
                        yield "\r\n"

                        for start, stop in r:
                            yield "--" + boundary
                            yield "\r\nContent-type: %s" % content_type
                            yield (
                                "\r\nContent-range: bytes %s-%s/%s\r\n\r\n" %
                                (start, stop - 1, c_len))
                            file.seek(start)
                            for chunk in file_generator_limited(
                                    file, stop - start):
                                yield chunk
                            yield "\r\n"
                        # Final boundary
                        yield "--" + boundary + "--"

                        # Apache compatibility:
                        yield "\r\n"

                    response.body = file_ranges()
            else:
                response.headers['Content-Length'] = c_len
                response.body = file
        else:
            response.headers['Content-Length'] = c_len
            response.body = file

        return response.body
Example #25
0
    def send_file_if_conditions_met(self, in_file, filename, n_bytes=None, skip_unhandled_extension=True, skip_duplicate=True, metadata=None, sha1=None):
        """Upload a file to the Overview server.

        If ``n_bytes is None or (skip_duplicate == True and sha1 is None)``,
        then ``in_file`` will be cached in memory. Otherwise, it will be
        streamed to the server, saving memory.

        :param io.BytesIO in_file: BytesIO containing the document.
        :param str filename: Filename to set in Overview.
        :param int n_bytes: Exact file size (or `None` to auto-calculate).
            Supply this and ``sha1`` (if applicable) to stream ``in_file`` to
            the server instead of caching it in memory.
        :param bool skip_unhandled_extension: if ``True`` (the default), do not
            upload this file if Overview doesn't support its filename extension
            (for instance, ``".dbf"``).
        :param bool skip_duplicate: if ``True`` (the default), do not upload
            this file if your api_token points to a document set that already
            contains a file whose sha1 hash is identical to this file's. Files
            that have been sent without a call to ``finish()`` will not be
            included in the check.
        :param dict metadata: Metadata to set on the document, or ``None``.
            The document set should have a metadata schema that corresponds to
            this document's metadata (or you can set the schema later).
        :param str sha1: SHA1 hash:to use in ``skip_duplicate()`` check, or
            ``None`` to calculate on the fly. If you set this and ``n_bytes``,
            this method will stream the file contents instead of caching them
            in memory.
        """
        if skip_unhandled_extension:
            # We go by filename, with a blacklist we know Overview doesn't handle (yet)
            path, ext = os.path.splitext(filename)
            if ext.lower() in ['.zip', '.msg', '.gif', '.jpg', '.png', '.tiff', '.tif', '.dbf']:
                self.logger.info('Skipping %s, Overview does not handle this format', filename)
                return

        if skip_duplicate:
            if sha1 is None:
                # Cache in_file bytes in memory so we can read it twice: once in
                # is_file_already_in_document_set(), and once below.
                in_file = io.BytesIO(in_file.read())
                sha1 = _calculate_sha1(in_file)
                in_file.seek(0)

            if self.is_file_already_in_document_set(in_file, sha1):
                self.logger.info('Skipping %s, already on server', filename)
                return

        if n_bytes is None:
            # Cache in_file bytes in memory so we can read it twice: once 
            # here, once below
            in_file = io.BytesIO(in_file.read())
            n_bytes = in_file.getbuffer().nbytes

        server_path = '/api/v1/files/{}'.format(uuid.uuid4())
        headers = {
            'Content-Disposition': rfc6266.build_header(filename),
            'Content-Length': str(n_bytes),
        }
        if metadata:
            headers['Overview-Document-Metadata-JSON'] = json.dumps(metadata, ensure_ascii=True)

        self.logger.info('Uploading %s…', filename)
        r = self._request('POST', server_path, headers=headers, data=in_file)
        r.raise_for_status()
        self.n_uploaded += 1
Example #26
0
def video_encodings_download(request, course_key_string):
    """
    Returns a CSV report containing the encoded video URLs for video uploads
    in the following format:

    Video ID,Name,Status,Profile1 URL,Profile2 URL
    aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4
    """
    course = _get_and_validate_course(course_key_string, request.user)

    if not course:
        return HttpResponseNotFound()

    def get_profile_header(profile):
        """Returns the column header string for the given profile's URLs"""
        # Translators: This is the header for a CSV file column
        # containing URLs for video encodings for the named profile
        # (e.g. desktop, mobile high quality, mobile low quality)
        return _("{profile_name} URL").format(profile_name=profile)

    profile_whitelist = VideoUploadConfig.get_profile_whitelist()

    videos = list(_get_videos(course))
    name_col = _("Name")
    duration_col = _("Duration")
    added_col = _("Date Added")
    video_id_col = _("Video ID")
    status_col = _("Status")
    profile_cols = [get_profile_header(profile) for profile in profile_whitelist]

    def make_csv_dict(video):
        """
        Makes a dictionary suitable for writing CSV output. This involves
        extracting the required items from the original video dict and
        converting all keys and values to UTF-8 encoded string objects,
        because the CSV module doesn't play well with unicode objects.
        """
        # Translators: This is listed as the duration for a video that has not
        # yet reached the point in its processing by the servers where its
        # duration is determined.
        duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending")
        ret = dict(
            [
                (name_col, video["client_video_id"]),
                (duration_col, duration_val),
                (added_col, video["created"].isoformat()),
                (video_id_col, video["edx_video_id"]),
                (status_col, video["status"]),
            ]
            + [
                (get_profile_header(encoded_video["profile"]), encoded_video["url"])
                for encoded_video in video["encoded_videos"]
                if encoded_video["profile"] in profile_whitelist
            ]
        )
        return {key.encode("utf-8"): value.encode("utf-8") for key, value in ret.items()}

    response = HttpResponse(content_type="text/csv")
    # Translators: This is the suggested filename when downloading the URL
    # listing for videos uploaded through Studio
    filename = _("{course}_video_urls").format(course=course.id.course)
    # See https://tools.ietf.org/html/rfc6266#appendix-D
    response["Content-Disposition"] = rfc6266.build_header(filename + ".csv", filename_compat="video_urls.csv")
    writer = csv.DictWriter(
        response,
        [
            col_name.encode("utf-8")
            for col_name in [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols
        ],
        dialect=csv.excel,
    )
    writer.writeheader()
    for video in videos:
        writer.writerow(make_csv_dict(video))
    return response
    def upload(self, post_file, uuid=None):
        now_iso = get_now().isoformat()
        filename = get_filename(post_file.filename)
        content_type = post_file.type
        in_file = post_file.file
        md5hash = self.compute_md5(in_file)
        if md5hash in self.forbidden_hash:
            LOGGER.warning("Forbidden file by hash {}".format(md5hash))
            raise StorageUploadError('forbidden_file ' + md5hash)

        if uuid is None:
            uuid = self.hash_to_uuid(md5hash)
            meta = dict(uuid=uuid, hash=md5hash, created=now_iso)
        else:
            meta = self.read_meta(uuid)
            if not compare_digest(meta['hash'], md5hash):
                raise HashInvalid(meta['hash'] + "/" + md5hash)

        key = self.uuid_to_file(uuid)
        path, name = self.file_path(key)
        if os.path.exists(name):
            meta = self.read_meta(uuid)
            if meta['filename'] != filename:
                if 'alternatives' not in meta:
                    meta['alternatives'] = list()
                meta['alternatives'].append({
                    'created': now_iso,
                    'filename': filename
                })
                self.save_meta(uuid, meta, overwrite=True)
            return uuid, md5hash, content_type, filename

        if self.check_forbidden(filename, content_type, in_file):
            LOGGER.warning("Forbidden file {} {} {} {}".format(
                filename, content_type, uuid, md5hash))
            raise StorageUploadError('forbidden_file ' + md5hash)

        meta['filename'] = filename
        meta['Content-Type'] = content_type
        meta['Content-Disposition'] = build_header(
            filename,
            disposition=self.disposition,
            filename_compat=quote(filename.encode('utf-8')))

        self.save_meta(uuid, meta, overwrite=True)

        in_file.seek(0)
        with open(name + '~', 'wb') as out_file:
            flock(out_file, LOCK_EX | LOCK_NB)
            copyfileobj(in_file, out_file)
        os.rename(name + '~', name)
        os.chmod(name, self.file_mode)

        try:
            if self.replica_apis:
                self.upload_to_replicas(post_file, uuid)
        except Exception as e:  # pragma: no cover
            LOGGER.error("Replica failed {}, remove file {} {}".format(
                e, uuid, md5hash))
            if self.require_replica_upload:
                os.rename(name, name + '~')
                raise StorageUploadError('replica_failed')

        return uuid, md5hash, content_type, filename
Example #28
0
 def roundtrip(filename):
     return parse_headers(build_header(filename)).filename_unsafe
Example #29
0
    return ajax_response(request, respmsg=_("Filters set activated"))


@login_required
@needs_mailbox()
def download_filters_set(request, name):
    sc = SieveClient(user=request.user.username, password=request.session["password"])
    try:
        script = sc.getscript(name)
    except SieveClientError, e:
        return ajax_response(request, "ko", respmsg=str(e))

    resp = HttpResponse(script)
    resp["Content-Type"] = "text/plain; charset=utf-8"
    resp["Content-Length"] = len(script)
    resp["Content-Disposition"] = build_header("%s.txt" % name)
    return resp


@login_required
@needs_mailbox()
def toggle_filter_state(request, setname, fname):
    sc = SieveClient(user=request.user.username, password=request.session["password"])
    if type(fname) is unicode:
        fname = fname.encode("utf-8")
    fset = sc.getscript(setname, format="fset")
    if fset.is_filter_disabled(fname):
        ret = fset.enablefilter(fname)
        newstate = _("yes")
        color = "green"
    else:
Example #30
0
    def __init__(self,
                 f,
                 request,
                 disposition='attachment',
                 cache_max_age=604800,
                 content_type=None,
                 content_encoding=None):
        """
        :param f: the ``UploadedFile`` file field value.
        :type f: :class:`depot.io.interfaces.StoredFile`

        :param request: Current request.
        :type request: :class:`pyramid.request.Request`

        :param disposition:
        :type disposition:

        :param cache_max_age: The number of seconds that should be used to HTTP
                              cache this response.

        :param content_type: The content_type of the response.

        :param content_encoding: The content_encoding of the response.
                                 It's generally safe to leave this set to
                                 ``None`` if you're serving a binary file.
                                 This argument will be ignored if you also
                                 leave ``content-type`` as ``None``.
        """

        if f.public_url:
            raise HTTPMovedPermanently(f.public_url)

        content_encoding, content_type = self._get_type_and_encoding(
            content_encoding, content_type, f)

        super(StoredFileResponse,
              self).__init__(conditional_response=True,
                             content_type=content_type,
                             content_encoding=content_encoding)

        app_iter = None
        if request is not None and \
                not get_settings()['kotti.depot_replace_wsgi_file_wrapper']:
            environ = request.environ
            if 'wsgi.file_wrapper' in environ:
                app_iter = environ['wsgi.file_wrapper'](f, _BLOCK_SIZE)
        if app_iter is None:
            app_iter = FileIter(f)
        self.app_iter = app_iter

        # assignment of content_length must come after assignment of app_iter
        self.content_length = f.content_length
        self.last_modified = f.last_modified

        if cache_max_age is not None:
            self.cache_expires = cache_max_age
            self.cache_control.public = True

        self.etag = self.generate_etag(f)
        self.content_disposition = rfc6266.build_header(
            f.filename,
            disposition=disposition,
            filename_compat=unidecode(f.filename))
Example #31
0
 def make_content_disposition_header(fn):
     return build_header(os.path.basename(fn)).decode('ascii')
Example #32
0
 def roundtrip(filename):
     return parse_headers(build_header(filename)).filename_unsafe
Example #33
0

@login_required
@needs_mailbox()
def download_filters_set(request, name):
    sc = SieveClient(user=request.user.username,
                     password=request.session["password"])
    try:
        script = sc.getscript(name)
    except SieveClientError, e:
        return ajax_response(request, "ko", respmsg=str(e))

    resp = HttpResponse(script)
    resp["Content-Type"] = "text/plain; charset=utf-8"
    resp["Content-Length"] = len(script)
    resp["Content-Disposition"] = build_header('%s.txt' % name)
    return resp


@login_required
@needs_mailbox()
def toggle_filter_state(request, setname, fname):
    sc = SieveClient(user=request.user.username,
                     password=request.session["password"])
    if type(fname) is unicode:
        fname = fname.encode("utf-8")
    try:
        fset = sc.getscript(setname, format="fset")
        if fset.is_filter_disabled(fname):
            ret = fset.enablefilter(fname)
            newstate = _("yes")
Example #34
0
def test_test_roundtrip():
    assert (build_header("test.txt", disposition='form-data', name="test1")
            == 'form-data; name=test1; filename=test.txt')

    assert (build_header(u"тест.txt", disposition='form-data', name="test2")
            == "form-data; name=test2; filename*=utf-8''%D1%82%D0%B5%D1%81%D1%82.txt")
Example #35
0
async def render(request: web.Request) -> web.Response:
    """HTTP POST route at /render for web nbconvert API.
    Return response of conversion with appropriate CONTENT-DISPOSITION
    """
    post_data = await request.post()

    try:
        notebook_field = post_data['notebook']
    except KeyError:
        return make_web_error_response(
            request, "Missing field",
            "Missing notebook [multipart file] field")

    if not notebook_field:
        return make_web_error_response(request, "Invalid field",
                                       "Notebook file multipart field empty")

    try:
        exporter_type = post_data['exporter']
    except KeyError:
        return make_web_error_response(request, "Missing field",
                                       "Missing exporter field")

    exporter_names = get_exporter_names()
    if exporter_type not in exporter_names:
        return make_web_error_response(
            request, "Invalid field",
            f"Invalid exporter {exporter_type!r}, must be one of "
            f"{exporter_names}")
    try:
        disposition = post_data['disposition']
    except KeyError:
        return make_web_error_response(request, "Missing field",
                                       "Missing disposition field")
    if disposition not in DISPOSITION_FIELDS:
        return make_web_error_response(
            request, "Invalid field",
            f"Invalid disposition {disposition!r}, must be one of {DISPOSITION_FIELDS}"
        )

    notebook_string = notebook_field.file.read()
    notebook_data = nbformat.reads(notebook_string, TO_VERSION)

    config = {
        'Exporter': {
            'preprocessors':
            ['nbconvert_http.preprocessors.TagExtractPreprocessor']
        },
        'TagExtractPreprocessor': {
            'extract_cell_tags': ['bibliography']
        }
    }

    # Only need to intercept template for the HTML API
    async with render_execution_context(exporter_type, config):
        loop = asyncio.get_event_loop()
        result = await loop.run_in_executor(pool, convert_notebook_sync,
                                            notebook_data, exporter_type,
                                            config)

    filename = f"result{result['resources']['output_extension']}"
    response = web.Response(body=result['body'],
                            headers=CIMultiDict({
                                'CONTENT-DISPOSITION':
                                rfc6266.build_header(filename, disposition)
                            }))
    response.content_type = result['mime-type']
    return response