Пример #1
0
 def ws_control_frame(self, opcode, data):
     if opcode in (PING, CLOSE):
         rcode = PONG if opcode == PING else CLOSE
         if opcode == CLOSE:
             self.ws_close_received = True
             self.stop_reading = True
             if data:
                 try:
                     close_code = unpack_from(b'!H', data)[0]
                 except struct_error:
                     data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be atleast two bytes'
                 else:
                     try:
                         utf8_decode(data[2:])
                     except ValueError:
                         data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be valid UTF-8'
                     else:
                         if close_code < 1000 or close_code in RESERVED_CLOSE_CODES or (1011 < close_code < 3000):
                             data = pack(b'!H', PROTOCOL_ERROR) + b'close code reserved'
             else:
                 close_code = NORMAL_CLOSE
                 data = pack(b'!H', close_code)
         f = ReadOnlyFileBuffer(create_frame(1, rcode, data))
         f.is_close_frame = opcode == CLOSE
         with self.cf_lock:
             self.control_frames.append(f)
     elif opcode == PONG:
         try:
             self.websocket_handler.handle_websocket_pong(self.websocket_connection_id, data)
         except Exception:
             self.log.exception('Error in PONG handler:')
     self.set_ws_state()
Пример #2
0
 def write_ranges(self, buf, ranges, event, first=False):
     r, range_part = next(ranges)
     if r is None:
         # EOF range part
         self.set_state(WRITE, self.write_buf, ReadOnlyFileBuffer(b'\r\n' + range_part))
     else:
         buf.seek(r.start)
         self.set_state(WRITE, self.write_range_part, ReadOnlyFileBuffer((b'' if first else b'\r\n') + range_part + b'\r\n'), buf, r.stop + 1, ranges)
Пример #3
0
 def websocket_close(self, code=NORMAL_CLOSE, reason=b''):
     if isinstance(reason, type('')):
         reason = reason.encode('utf-8')
     self.stop_reading = True
     reason = reason[:123]
     if code is None and not reason:
         f = ReadOnlyFileBuffer(create_frame(1, CLOSE, b''))
     else:
         f = ReadOnlyFileBuffer(create_frame(1, CLOSE, pack(b'!H', code) + reason))
     f.is_close_frame = True
     with self.cf_lock:
         self.control_frames.append(f)
     self.set_ws_state()
Пример #4
0
 def write_iter(self, output, event):
     chunk = next(output)
     if chunk is None:
         self.set_state(WRITE, self.write_chunk, ReadOnlyFileBuffer(b'0\r\n\r\n'), output, last=True)
     else:
         if chunk:
             if not isinstance(chunk, bytes):
                 chunk = chunk.encode('utf-8')
             chunk = ('%X\r\n' % len(chunk)).encode('ascii') + chunk + b'\r\n'
             self.set_state(WRITE, self.write_chunk, ReadOnlyFileBuffer(chunk), output)
         else:
             # Empty chunk, ignore it
             self.write_iter(output, event)
Пример #5
0
 def __init__(self, buf, mask=None, chunk_size=None):
     self.buf, self.data_type, self.mask = buf, BINARY, mask
     if isinstance(buf, type('')):
         self.buf, self.data_type = ReadOnlyFileBuffer(buf.encode('utf-8')), TEXT
     elif isinstance(buf, bytes):
         self.buf = ReadOnlyFileBuffer(buf)
     buf = self.buf
     self.chunk_size = chunk_size or SEND_CHUNK_SIZE
     try:
         pos = buf.tell()
         buf.seek(0, os.SEEK_END)
         self.size = buf.tell() - pos
         buf.seek(pos)
     except Exception:
         self.size = None
     self.first_frame_created = self.exhausted = False
Пример #6
0
    def simple_response(self, status_code, msg='', close_after_response=True, extra_headers=None):
        if self.response_protocol is HTTP1:
            # HTTP/1.0 has no 413/414/303 codes
            status_code = {
                httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST,
                httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST,
                httplib.SEE_OTHER:httplib.FOUND
            }.get(status_code, status_code)

        self.close_after_response = close_after_response
        msg = msg.encode('utf-8')
        ct = 'http' if self.method == 'TRACE' else 'plain'
        buf = [
            '%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]),
            "Content-Length: %s" % len(msg),
            "Content-Type: text/%s; charset=UTF-8" % ct,
            "Date: " + http_date(),
        ]
        if self.close_after_response and self.response_protocol is HTTP11:
            buf.append("Connection: close")
        if extra_headers is not None:
            for h, v in extra_headers.iteritems():
                buf.append('%s: %s' % (h, v))
        buf.append('')
        buf = [(x + '\r\n').encode('ascii') for x in buf]
        if self.method != 'HEAD':
            buf.append(msg)
        response_data = b''.join(buf)
        self.log_access(status_code=status_code, response_size=len(response_data))
        self.response_ready(ReadOnlyFileBuffer(response_data))
Пример #7
0
    def finalize_headers(self, inheaders):
        upgrade = inheaders.get('Upgrade', '')
        key = inheaders.get('Sec-WebSocket-Key', None)
        conn = {
            x.strip().lower()
            for x in inheaders.get('Connection', '').split(',')
        }
        if key is None or upgrade.lower(
        ) != 'websocket' or 'upgrade' not in conn:
            return HTTPConnection.finalize_headers(self, inheaders)
        ver = inheaders.get('Sec-WebSocket-Version', 'Unknown')
        try:
            ver_ok = int(ver) >= 13
        except Exception:
            ver_ok = False
        if not ver_ok:
            return self.simple_response(
                http_client.BAD_REQUEST,
                'Unsupported WebSocket protocol version: %s' % ver)
        if self.method != 'GET':
            return self.simple_response(
                http_client.BAD_REQUEST,
                'Invalid WebSocket method: %s' % self.method)

        response = HANDSHAKE_STR % as_base64_unicode(
            sha1((key + GUID_STR).encode('utf-8')).digest())
        self.optimize_for_sending_packet()
        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        self.set_state(WRITE, self.upgrade_connection_to_ws,
                       ReadOnlyFileBuffer(response.encode('ascii')), inheaders)
Пример #8
0
 def send_websocket_ping(self, data=b''):
     ''' Send a PING to the remote client, it should reply with a PONG which
     will be sent to the handle_websocket_pong callback in your handler. '''
     if isinstance(data, type('')):
         data = data.encode('utf-8')
     frame = create_frame(True, PING, data)
     with self.cf_lock:
         self.control_frames.append(ReadOnlyFileBuffer(frame))
Пример #9
0
def identify(src):
    ''' Recognize file format and sizes. Returns format, width, height. width
    and height will be -1 if not found and fmt will be None if the image is not
    recognized. '''
    width = height = -1

    if isinstance(src, type('')):
        stream = lopen(src, 'rb')
    elif isinstance(src, bytes):
        stream = ReadOnlyFileBuffer(src)
    else:
        stream = src

    pos = stream.tell()
    head = stream.read(HSIZE)
    stream.seek(pos)
    fmt = what(None, head)

    if fmt in {'jpeg', 'gif', 'png', 'jpeg2000'}:
        size = len(head)
        if fmt == 'png':
            # PNG
            s = head[16:24] if size >= 24 and head[12:16] == b'IHDR' else head[
                8:16]
            try:
                width, height = unpack(b">LL", s)
            except error:
                return fmt, width, height
        elif fmt == 'jpeg':
            # JPEG
            pos = stream.tell()
            try:
                height, width = jpeg_dimensions(stream)
            except Exception:
                return fmt, width, height
            finally:
                stream.seek(pos)
        elif fmt == 'gif':
            # GIF
            try:
                width, height = unpack(b"<HH", head[6:10])
            except error:
                return fmt, width, height
        elif size >= 56 and fmt == 'jpeg2000':
            # JPEG2000
            try:
                height, width = unpack(b'>LL', head[48:56])
            except error:
                return fmt, width, height
    return fmt, width, height
Пример #10
0
def identify(src):
    ''' Recognize file format and sizes. Returns format, width, height. width
    and height will be -1 if not found and fmt will be None if the image is not
    recognized. '''
    needs_close = False

    if isinstance(src, str):
        stream = lopen(src, 'rb')
        needs_close = True
    elif isinstance(src, bytes):
        stream = ReadOnlyFileBuffer(src)
    else:
        stream = src
    try:
        return _identify(stream)
    finally:
        if needs_close:
            stream.close()
Пример #11
0
 def send_websocket_frame(self, data, is_first=True, is_last=True):
     ''' Useful for streaming handlers that want to break up messages into
     frames themselves. Note that these frames will be interleaved with
     control frames, so they should not be too large. '''
     opcode = (TEXT if isinstance(data, type('')) else BINARY) if is_first else CONTINUATION
     fin = 1 if is_last else 0
     frame = create_frame(fin, opcode, data)
     with self.cf_lock:
         self.control_frames.append(ReadOnlyFileBuffer(frame))
Пример #12
0
 def create_frame(self):
     if self.exhausted:
         return None
     buf = self.buf
     raw = buf.read(self.chunk_size)
     has_more = True if self.size is None else self.size > buf.tell()
     fin = 0 if has_more and raw else 1
     opcode = 0 if self.first_frame_created else self.data_type
     self.first_frame_created, self.exhausted = True, bool(fin)
     return ReadOnlyFileBuffer(create_frame(fin, opcode, raw, self.mask))
Пример #13
0
    def job_done(self, ok, result):
        if not ok:
            etype, e, tb = result
            if isinstance(e, HTTPSimpleResponse):
                eh = {}
                if e.location:
                    eh['Location'] = e.location
                if e.authenticate:
                    eh['WWW-Authenticate'] = e.authenticate
                if e.log:
                    self.log.warn(e.log)
                return self.simple_response(e.http_code, msg=e.message or '', close_after_response=e.close_connection, extra_headers=eh)
            reraise(etype, e, tb)

        data, output = result
        output = self.finalize_output(output, data, self.method is HTTP1)
        if output is None:
            return
        outheaders = data.outheaders

        outheaders.set('Date', http_date(), replace_all=True)
        outheaders.set('Server', 'calibre %s' % __version__, replace_all=True)
        keep_alive = not self.close_after_response and self.opts.timeout > 0
        if keep_alive:
            outheaders.set('Keep-Alive', 'timeout=%d' % int(self.opts.timeout))
        if 'Connection' not in outheaders:
            if self.response_protocol is HTTP11:
                if self.close_after_response:
                    outheaders.set('Connection', 'close')
            else:
                if not self.close_after_response:
                    outheaders.set('Connection', 'Keep-Alive')

        ct = outheaders.get('Content-Type', '')
        if ct.startswith('text/') and 'charset=' not in ct:
            outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True)

        buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]]
        for header, value in sorted(outheaders.iteritems(), key=itemgetter(0)):
            buf.append('%s: %s' % (header, value))
        for morsel in data.outcookie.itervalues():
            morsel['version'] = '1'
            x = morsel.output()
            if isinstance(x, bytes):
                x = x.decode('ascii')
            buf.append(x)
        buf.append('')
        response_data = ReadOnlyFileBuffer(b''.join((x + '\r\n').encode('ascii') for x in buf))
        if self.access_log is not None:
            sz = outheaders.get('Content-Length')
            if sz is not None:
                sz = int(sz) + response_data.sz
            self.log_access(status_code=data.status_code, response_size=sz, username=data.username)
        self.response_ready(response_data, output=output)
Пример #14
0
def dynamic_output(output, outheaders, etag=None):
    if isinstance(output, bytes):
        data = output
    else:
        data = output.encode('utf-8')
        ct = outheaders.get('Content-Type')
        if not ct:
            outheaders.set('Content-Type', 'text/plain; charset=UTF-8', replace_all=True)
    ans = ReadableOutput(ReadOnlyFileBuffer(data), etag=etag)
    ans.accept_ranges = False
    return ans
Пример #15
0
def cdb_set_fields(ctx, rd, book_id, library_id):
    db = get_db(ctx, rd, library_id)
    if ctx.restriction_for(rd, db):
        raise HTTPForbidden('Cannot use the set fields interface with a user who has per library restrictions')
    data = load_payload_data(rd)
    try:
        changes, loaded_book_ids = data['changes'], frozenset(map(int, data.get('loaded_book_ids', ())))
        all_dirtied = bool(data.get('all_dirtied'))
        if not isinstance(changes, dict):
            raise TypeError('changes must be a dict')
    except Exception:
        raise HTTPBadRequest(
        '''Data must be of the form {'changes': {'title': 'New Title', ...}, 'loaded_book_ids':[book_id1, book_id2, ...]'}''')
    dirtied = set()
    cdata = changes.pop('cover', False)
    if cdata is not False:
        if cdata is not None:
            try:
                cdata = from_base64_bytes(cdata.split(',', 1)[-1])
            except Exception:
                raise HTTPBadRequest('Cover data is not valid base64 encoded data')
            try:
                fmt = what(None, cdata)
            except Exception:
                fmt = None
            if fmt not in ('jpeg', 'png'):
                raise HTTPBadRequest('Cover data must be either JPEG or PNG')
        dirtied |= db.set_cover({book_id: cdata})

    added_formats = changes.pop('added_formats', False)
    if added_formats:
        for data in added_formats:
            try:
                fmt = data['ext'].upper()
            except Exception:
                raise HTTPBadRequest('Format has no extension')
            if fmt:
                try:
                    fmt_data = from_base64_bytes(data['data_url'].split(',', 1)[-1])
                except Exception:
                    raise HTTPBadRequest('Format data is not valid base64 encoded data')
                if db.add_format(book_id, fmt, ReadOnlyFileBuffer(fmt_data)):
                    dirtied.add(book_id)
    removed_formats = changes.pop('removed_formats', False)
    if removed_formats:
        db.remove_formats({book_id: list(removed_formats)})
        dirtied.add(book_id)

    for field, value in iteritems(changes):
        dirtied |= db.set_field(field, {book_id: value})
    ctx.notify_changes(db.backend.library_path, metadata(dirtied))
    all_ids = dirtied if all_dirtied else (dirtied & loaded_book_ids)
    all_ids |= {book_id}
    return {bid: book_as_json(db, bid) for bid in all_ids}
Пример #16
0
def identify(src):
    ''' Recognize file format and sizes. Returns format, width, height. width
    and height will be -1 if not found and fmt will be None if the image is not
    recognized. '''
    width = height = -1

    if isinstance(src, type('')):
        stream = lopen(src, 'rb')
    elif isinstance(src, bytes):
        stream = ReadOnlyFileBuffer(src)
    else:
        stream = src

    pos = stream.tell()
    head = stream.read(HSIZE)
    stream.seek(pos)
    fmt = what(None, head)

    if fmt in {'jpeg', 'gif', 'png', 'jpeg2000'}:
        size = len(head)
        if fmt == 'png':
            # PNG
            s = head[16:24] if size >= 24 and head[12:16] == b'IHDR' else head[8:16]
            try:
                width, height = unpack(b">LL", s)
            except error:
                return fmt, width, height
        elif fmt == 'jpeg':
            # JPEG
            pos = stream.tell()
            try:
                height, width = jpeg_dimensions(stream)
            except Exception:
                return fmt, width, height
            finally:
                stream.seek(pos)
        elif fmt == 'gif':
            # GIF
            try:
                width, height = unpack(b"<HH", head[6:10])
            except error:
                return fmt, width, height
        elif size >= 56 and fmt == 'jpeg2000':
            # JPEG2000
            try:
                height, width = unpack(b'>LL', head[48:56])
            except error:
                return fmt, width, height
    return fmt, width, height
Пример #17
0
def encode_jpeg(file_path, quality=80):
    from calibre.utils.speedups import ReadOnlyFileBuffer
    quality = max(0, min(100, int(quality)))
    exe = get_exe_path('cjpeg')
    cmd = [exe] + '-optimize -progressive -maxmemory 100M -quality'.split() + [str(quality)]
    img = QImage()
    if not img.load(file_path):
        raise ValueError('%s is not a valid image file' % file_path)
    ba = QByteArray()
    buf = QBuffer(ba)
    buf.open(QBuffer.WriteOnly)
    if not img.save(buf, 'PPM'):
        raise ValueError('Failed to export image to PPM')
    return run_optimizer(file_path, cmd, as_filter=True, input_data=ReadOnlyFileBuffer(ba.data()))
Пример #18
0
    def finalize_output(self, output, request, is_http1):
        none_match = parse_if_none_match(
            request.inheaders.get('If-None-Match', ''))
        if isinstance(output, ETaggedDynamicOutput):
            matched = '*' in none_match or (output.etag
                                            and output.etag in none_match)
            if matched:
                if self.method in ('GET', 'HEAD'):
                    self.send_not_modified(output.etag)
                else:
                    self.simple_response(httplib.PRECONDITION_FAILED)
                return

        opts = self.opts
        outheaders = request.outheaders
        stat_result = file_metadata(output)
        if stat_result is not None:
            output = filesystem_file_output(output, outheaders, stat_result)
            if 'Content-Type' not in outheaders:
                mt = guess_type(output.name)[0]
                if mt:
                    if mt in {
                            'text/plain', 'text/html',
                            'application/javascript', 'text/css'
                    }:
                        mt += '; charset=UTF-8'
                    outheaders['Content-Type'] = mt
        elif isinstance(output, (bytes, type(''))):
            output = dynamic_output(output, outheaders)
        elif hasattr(output, 'read'):
            output = ReadableOutput(output)
        elif isinstance(output, StaticOutput):
            output = ReadableOutput(ReadOnlyFileBuffer(output.data),
                                    etag=output.etag,
                                    content_length=output.content_length)
        elif isinstance(output, ETaggedDynamicOutput):
            output = dynamic_output(output(), outheaders, etag=output.etag)
        else:
            output = GeneratedOutput(output)
        ct = outheaders.get('Content-Type', '').partition(';')[0]
        compressible = (not ct or ct.startswith('text/')
                        or ct.startswith('image/svg')
                        or ct.partition(';')[0] in COMPRESSIBLE_TYPES)
        compressible = (compressible and request.status_code == httplib.OK and
                        (opts.compress_min_size > -1
                         and output.content_length >= opts.compress_min_size)
                        and acceptable_encoding(
                            request.inheaders.get('Accept-Encoding', ''))
                        and not is_http1)
        accept_ranges = (not compressible and output.accept_ranges is not None
                         and request.status_code == httplib.OK
                         and not is_http1)
        ranges = get_ranges(
            request.inheaders.get('Range'), output.content_length
        ) if output.accept_ranges and self.method in ('GET', 'HEAD') else None
        if_range = (request.inheaders.get('If-Range') or '').strip()
        if if_range and if_range != output.etag:
            ranges = None
        if ranges is not None and not ranges:
            return self.send_range_not_satisfiable(output.content_length)

        for header in ('Accept-Ranges', 'Content-Encoding',
                       'Transfer-Encoding', 'ETag', 'Content-Length'):
            outheaders.pop(header, all=True)

        matched = '*' in none_match or (output.etag
                                        and output.etag in none_match)
        if matched:
            if self.method in ('GET', 'HEAD'):
                self.send_not_modified(output.etag)
            else:
                self.simple_response(httplib.PRECONDITION_FAILED)
            return

        output.ranges = None

        if output.etag and self.method in ('GET', 'HEAD'):
            outheaders.set('ETag', output.etag, replace_all=True)
        if accept_ranges:
            outheaders.set('Accept-Ranges', 'bytes', replace_all=True)
        if compressible and not ranges:
            outheaders.set('Content-Encoding', 'gzip', replace_all=True)
            if getattr(output, 'content_length', None):
                outheaders.set('Calibre-Uncompressed-Length',
                               '%d' % output.content_length)
            output = GeneratedOutput(compress_readable_output(output.src_file),
                                     etag=output.etag)
        if output.content_length is not None and not compressible and not ranges:
            outheaders.set('Content-Length',
                           '%d' % output.content_length,
                           replace_all=True)

        if compressible or output.content_length is None:
            outheaders.set('Transfer-Encoding', 'chunked', replace_all=True)

        if ranges:
            if len(ranges) == 1:
                r = ranges[0]
                outheaders.set('Content-Length',
                               '%d' % r.size,
                               replace_all=True)
                outheaders.set('Content-Range',
                               'bytes %d-%d/%d' %
                               (r.start, r.stop, output.content_length),
                               replace_all=True)
                output.ranges = r
            else:
                range_parts = get_range_parts(ranges,
                                              outheaders.get('Content-Type'),
                                              output.content_length)
                size = sum(map(len, range_parts)) + sum(r.size + 4
                                                        for r in ranges)
                outheaders.set('Content-Length', '%d' % size, replace_all=True)
                outheaders.set('Content-Type',
                               'multipart/byteranges; boundary=' +
                               MULTIPART_SEPARATOR,
                               replace_all=True)
                output.ranges = zip_longest(ranges, range_parts)
            request.status_code = httplib.PARTIAL_CONTENT
        return output
Пример #19
0
def header_list_to_file(buf):  # {{{
    buf.append('')
    return ReadOnlyFileBuffer(b''.join(
        (x + '\r\n').encode('ascii') for x in buf))