def ws_control_frame(self, opcode, data):
if opcode in (PING, CLOSE):
rcode = PONG if opcode == PING else CLOSE
if opcode == CLOSE:
self.ws_close_received = True
self.stop_reading = True
if data:
try:
close_code = unpack_from(b'!H', data)[0]
except struct_error:
data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be atleast two bytes'
else:
try:
utf8_decode(data[2:])
except ValueError:
data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be valid UTF-8'
else:
if close_code < 1000 or close_code in RESERVED_CLOSE_CODES or (1011 < close_code < 3000):
data = pack(b'!H', PROTOCOL_ERROR) + b'close code reserved'
else:
close_code = NORMAL_CLOSE
data = pack(b'!H', close_code)
f = ReadOnlyFileBuffer(create_frame(1, rcode, data))
f.is_close_frame = opcode == CLOSE
with self.cf_lock:
self.control_frames.append(f)
elif opcode == PONG:
try:
self.websocket_handler.handle_websocket_pong(self.websocket_connection_id, data)
except Exception:
self.log.exception('Error in PONG handler:')
self.set_ws_state()
def write_ranges(self, buf, ranges, event, first=False):
r, range_part = next(ranges)
if r is None:
# EOF range part
self.set_state(WRITE, self.write_buf, ReadOnlyFileBuffer(b'\r\n' + range_part))
else:
buf.seek(r.start)
self.set_state(WRITE, self.write_range_part, ReadOnlyFileBuffer((b'' if first else b'\r\n') + range_part + b'\r\n'), buf, r.stop + 1, ranges)
def websocket_close(self, code=NORMAL_CLOSE, reason=b''):
if isinstance(reason, type('')):
reason = reason.encode('utf-8')
self.stop_reading = True
reason = reason[:123]
if code is None and not reason:
f = ReadOnlyFileBuffer(create_frame(1, CLOSE, b''))
else:
f = ReadOnlyFileBuffer(create_frame(1, CLOSE, pack(b'!H', code) + reason))
f.is_close_frame = True
with self.cf_lock:
self.control_frames.append(f)
self.set_ws_state()
def write_iter(self, output, event):
chunk = next(output)
if chunk is None:
self.set_state(WRITE, self.write_chunk, ReadOnlyFileBuffer(b'0\r\n\r\n'), output, last=True)
else:
if chunk:
if not isinstance(chunk, bytes):
chunk = chunk.encode('utf-8')
chunk = ('%X\r\n' % len(chunk)).encode('ascii') + chunk + b'\r\n'
self.set_state(WRITE, self.write_chunk, ReadOnlyFileBuffer(chunk), output)
else:
# Empty chunk, ignore it
self.write_iter(output, event)
def __init__(self, buf, mask=None, chunk_size=None):
self.buf, self.data_type, self.mask = buf, BINARY, mask
if isinstance(buf, type('')):
self.buf, self.data_type = ReadOnlyFileBuffer(buf.encode('utf-8')), TEXT
elif isinstance(buf, bytes):
self.buf = ReadOnlyFileBuffer(buf)
buf = self.buf
self.chunk_size = chunk_size or SEND_CHUNK_SIZE
try:
pos = buf.tell()
buf.seek(0, os.SEEK_END)
self.size = buf.tell() - pos
buf.seek(pos)
except Exception:
self.size = None
self.first_frame_created = self.exhausted = False
def simple_response(self, status_code, msg='', close_after_response=True, extra_headers=None):
if self.response_protocol is HTTP1:
# HTTP/1.0 has no 413/414/303 codes
status_code = {
httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST,
httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST,
httplib.SEE_OTHER:httplib.FOUND
}.get(status_code, status_code)
self.close_after_response = close_after_response
msg = msg.encode('utf-8')
ct = 'http' if self.method == 'TRACE' else 'plain'
buf = [
'%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]),
"Content-Length: %s" % len(msg),
"Content-Type: text/%s; charset=UTF-8" % ct,
"Date: " + http_date(),
]
if self.close_after_response and self.response_protocol is HTTP11:
buf.append("Connection: close")
if extra_headers is not None:
for h, v in extra_headers.iteritems():
buf.append('%s: %s' % (h, v))
buf.append('')
buf = [(x + '\r\n').encode('ascii') for x in buf]
if self.method != 'HEAD':
buf.append(msg)
response_data = b''.join(buf)
self.log_access(status_code=status_code, response_size=len(response_data))
self.response_ready(ReadOnlyFileBuffer(response_data))
def finalize_headers(self, inheaders):
upgrade = inheaders.get('Upgrade', '')
key = inheaders.get('Sec-WebSocket-Key', None)
conn = {
x.strip().lower()
for x in inheaders.get('Connection', '').split(',')
}
if key is None or upgrade.lower(
) != 'websocket' or 'upgrade' not in conn:
return HTTPConnection.finalize_headers(self, inheaders)
ver = inheaders.get('Sec-WebSocket-Version', 'Unknown')
try:
ver_ok = int(ver) >= 13
except Exception:
ver_ok = False
if not ver_ok:
return self.simple_response(
http_client.BAD_REQUEST,
'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET':
return self.simple_response(
http_client.BAD_REQUEST,
'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % as_base64_unicode(
sha1((key + GUID_STR).encode('utf-8')).digest())
self.optimize_for_sending_packet()
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.set_state(WRITE, self.upgrade_connection_to_ws,
ReadOnlyFileBuffer(response.encode('ascii')), inheaders)
def send_websocket_ping(self, data=b''):
''' Send a PING to the remote client, it should reply with a PONG which
will be sent to the handle_websocket_pong callback in your handler. '''
if isinstance(data, type('')):
data = data.encode('utf-8')
frame = create_frame(True, PING, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def identify(src):
''' Recognize file format and sizes. Returns format, width, height. width
and height will be -1 if not found and fmt will be None if the image is not
recognized. '''
width = height = -1
if isinstance(src, type('')):
stream = lopen(src, 'rb')
elif isinstance(src, bytes):
stream = ReadOnlyFileBuffer(src)
else:
stream = src
pos = stream.tell()
head = stream.read(HSIZE)
stream.seek(pos)
fmt = what(None, head)
if fmt in {'jpeg', 'gif', 'png', 'jpeg2000'}:
size = len(head)
if fmt == 'png':
# PNG
s = head[16:24] if size >= 24 and head[12:16] == b'IHDR' else head[
8:16]
try:
width, height = unpack(b">LL", s)
except error:
return fmt, width, height
elif fmt == 'jpeg':
# JPEG
pos = stream.tell()
try:
height, width = jpeg_dimensions(stream)
except Exception:
return fmt, width, height
finally:
stream.seek(pos)
elif fmt == 'gif':
# GIF
try:
width, height = unpack(b"<HH", head[6:10])
except error:
return fmt, width, height
elif size >= 56 and fmt == 'jpeg2000':
# JPEG2000
try:
height, width = unpack(b'>LL', head[48:56])
except error:
return fmt, width, height
return fmt, width, height
def identify(src):
''' Recognize file format and sizes. Returns format, width, height. width
and height will be -1 if not found and fmt will be None if the image is not
recognized. '''
needs_close = False
if isinstance(src, str):
stream = lopen(src, 'rb')
needs_close = True
elif isinstance(src, bytes):
stream = ReadOnlyFileBuffer(src)
else:
stream = src
try:
return _identify(stream)
finally:
if needs_close:
stream.close()
def send_websocket_frame(self, data, is_first=True, is_last=True):
''' Useful for streaming handlers that want to break up messages into
frames themselves. Note that these frames will be interleaved with
control frames, so they should not be too large. '''
opcode = (TEXT if isinstance(data, type('')) else BINARY) if is_first else CONTINUATION
fin = 1 if is_last else 0
frame = create_frame(fin, opcode, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def create_frame(self):
if self.exhausted:
return None
buf = self.buf
raw = buf.read(self.chunk_size)
has_more = True if self.size is None else self.size > buf.tell()
fin = 0 if has_more and raw else 1
opcode = 0 if self.first_frame_created else self.data_type
self.first_frame_created, self.exhausted = True, bool(fin)
return ReadOnlyFileBuffer(create_frame(fin, opcode, raw, self.mask))
def job_done(self, ok, result):
if not ok:
etype, e, tb = result
if isinstance(e, HTTPSimpleResponse):
eh = {}
if e.location:
eh['Location'] = e.location
if e.authenticate:
eh['WWW-Authenticate'] = e.authenticate
if e.log:
self.log.warn(e.log)
return self.simple_response(e.http_code, msg=e.message or '', close_after_response=e.close_connection, extra_headers=eh)
reraise(etype, e, tb)
data, output = result
output = self.finalize_output(output, data, self.method is HTTP1)
if output is None:
return
outheaders = data.outheaders
outheaders.set('Date', http_date(), replace_all=True)
outheaders.set('Server', 'calibre %s' % __version__, replace_all=True)
keep_alive = not self.close_after_response and self.opts.timeout > 0
if keep_alive:
outheaders.set('Keep-Alive', 'timeout=%d' % int(self.opts.timeout))
if 'Connection' not in outheaders:
if self.response_protocol is HTTP11:
if self.close_after_response:
outheaders.set('Connection', 'close')
else:
if not self.close_after_response:
outheaders.set('Connection', 'Keep-Alive')
ct = outheaders.get('Content-Type', '')
if ct.startswith('text/') and 'charset=' not in ct:
outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True)
buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]]
for header, value in sorted(outheaders.iteritems(), key=itemgetter(0)):
buf.append('%s: %s' % (header, value))
for morsel in data.outcookie.itervalues():
morsel['version'] = '1'
x = morsel.output()
if isinstance(x, bytes):
x = x.decode('ascii')
buf.append(x)
buf.append('')
response_data = ReadOnlyFileBuffer(b''.join((x + '\r\n').encode('ascii') for x in buf))
if self.access_log is not None:
sz = outheaders.get('Content-Length')
if sz is not None:
sz = int(sz) + response_data.sz
self.log_access(status_code=data.status_code, response_size=sz, username=data.username)
self.response_ready(response_data, output=output)
def dynamic_output(output, outheaders, etag=None):
if isinstance(output, bytes):
data = output
else:
data = output.encode('utf-8')
ct = outheaders.get('Content-Type')
if not ct:
outheaders.set('Content-Type', 'text/plain; charset=UTF-8', replace_all=True)
ans = ReadableOutput(ReadOnlyFileBuffer(data), etag=etag)
ans.accept_ranges = False
return ans
def cdb_set_fields(ctx, rd, book_id, library_id):
db = get_db(ctx, rd, library_id)
if ctx.restriction_for(rd, db):
raise HTTPForbidden('Cannot use the set fields interface with a user who has per library restrictions')
data = load_payload_data(rd)
try:
changes, loaded_book_ids = data['changes'], frozenset(map(int, data.get('loaded_book_ids', ())))
all_dirtied = bool(data.get('all_dirtied'))
if not isinstance(changes, dict):
raise TypeError('changes must be a dict')
except Exception:
raise HTTPBadRequest(
'''Data must be of the form {'changes': {'title': 'New Title', ...}, 'loaded_book_ids':[book_id1, book_id2, ...]'}''')
dirtied = set()
cdata = changes.pop('cover', False)
if cdata is not False:
if cdata is not None:
try:
cdata = from_base64_bytes(cdata.split(',', 1)[-1])
except Exception:
raise HTTPBadRequest('Cover data is not valid base64 encoded data')
try:
fmt = what(None, cdata)
except Exception:
fmt = None
if fmt not in ('jpeg', 'png'):
raise HTTPBadRequest('Cover data must be either JPEG or PNG')
dirtied |= db.set_cover({book_id: cdata})
added_formats = changes.pop('added_formats', False)
if added_formats:
for data in added_formats:
try:
fmt = data['ext'].upper()
except Exception:
raise HTTPBadRequest('Format has no extension')
if fmt:
try:
fmt_data = from_base64_bytes(data['data_url'].split(',', 1)[-1])
except Exception:
raise HTTPBadRequest('Format data is not valid base64 encoded data')
if db.add_format(book_id, fmt, ReadOnlyFileBuffer(fmt_data)):
dirtied.add(book_id)
removed_formats = changes.pop('removed_formats', False)
if removed_formats:
db.remove_formats({book_id: list(removed_formats)})
dirtied.add(book_id)
for field, value in iteritems(changes):
dirtied |= db.set_field(field, {book_id: value})
ctx.notify_changes(db.backend.library_path, metadata(dirtied))
all_ids = dirtied if all_dirtied else (dirtied & loaded_book_ids)
all_ids |= {book_id}
return {bid: book_as_json(db, bid) for bid in all_ids}
def identify(src):
''' Recognize file format and sizes. Returns format, width, height. width
and height will be -1 if not found and fmt will be None if the image is not
recognized. '''
width = height = -1
if isinstance(src, type('')):
stream = lopen(src, 'rb')
elif isinstance(src, bytes):
stream = ReadOnlyFileBuffer(src)
else:
stream = src
pos = stream.tell()
head = stream.read(HSIZE)
stream.seek(pos)
fmt = what(None, head)
if fmt in {'jpeg', 'gif', 'png', 'jpeg2000'}:
size = len(head)
if fmt == 'png':
# PNG
s = head[16:24] if size >= 24 and head[12:16] == b'IHDR' else head[8:16]
try:
width, height = unpack(b">LL", s)
except error:
return fmt, width, height
elif fmt == 'jpeg':
# JPEG
pos = stream.tell()
try:
height, width = jpeg_dimensions(stream)
except Exception:
return fmt, width, height
finally:
stream.seek(pos)
elif fmt == 'gif':
# GIF
try:
width, height = unpack(b"<HH", head[6:10])
except error:
return fmt, width, height
elif size >= 56 and fmt == 'jpeg2000':
# JPEG2000
try:
height, width = unpack(b'>LL', head[48:56])
except error:
return fmt, width, height
return fmt, width, height
def encode_jpeg(file_path, quality=80):
from calibre.utils.speedups import ReadOnlyFileBuffer
quality = max(0, min(100, int(quality)))
exe = get_exe_path('cjpeg')
cmd = [exe] + '-optimize -progressive -maxmemory 100M -quality'.split() + [str(quality)]
img = QImage()
if not img.load(file_path):
raise ValueError('%s is not a valid image file' % file_path)
ba = QByteArray()
buf = QBuffer(ba)
buf.open(QBuffer.WriteOnly)
if not img.save(buf, 'PPM'):
raise ValueError('Failed to export image to PPM')
return run_optimizer(file_path, cmd, as_filter=True, input_data=ReadOnlyFileBuffer(ba.data()))
def finalize_output(self, output, request, is_http1):
none_match = parse_if_none_match(
request.inheaders.get('If-None-Match', ''))
if isinstance(output, ETaggedDynamicOutput):
matched = '*' in none_match or (output.etag
and output.etag in none_match)
if matched:
if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag)
else:
self.simple_response(httplib.PRECONDITION_FAILED)
return
opts = self.opts
outheaders = request.outheaders
stat_result = file_metadata(output)
if stat_result is not None:
output = filesystem_file_output(output, outheaders, stat_result)
if 'Content-Type' not in outheaders:
mt = guess_type(output.name)[0]
if mt:
if mt in {
'text/plain', 'text/html',
'application/javascript', 'text/css'
}:
mt += '; charset=UTF-8'
outheaders['Content-Type'] = mt
elif isinstance(output, (bytes, type(''))):
output = dynamic_output(output, outheaders)
elif hasattr(output, 'read'):
output = ReadableOutput(output)
elif isinstance(output, StaticOutput):
output = ReadableOutput(ReadOnlyFileBuffer(output.data),
etag=output.etag,
content_length=output.content_length)
elif isinstance(output, ETaggedDynamicOutput):
output = dynamic_output(output(), outheaders, etag=output.etag)
else:
output = GeneratedOutput(output)
ct = outheaders.get('Content-Type', '').partition(';')[0]
compressible = (not ct or ct.startswith('text/')
or ct.startswith('image/svg')
or ct.partition(';')[0] in COMPRESSIBLE_TYPES)
compressible = (compressible and request.status_code == httplib.OK and
(opts.compress_min_size > -1
and output.content_length >= opts.compress_min_size)
and acceptable_encoding(
request.inheaders.get('Accept-Encoding', ''))
and not is_http1)
accept_ranges = (not compressible and output.accept_ranges is not None
and request.status_code == httplib.OK
and not is_http1)
ranges = get_ranges(
request.inheaders.get('Range'), output.content_length
) if output.accept_ranges and self.method in ('GET', 'HEAD') else None
if_range = (request.inheaders.get('If-Range') or '').strip()
if if_range and if_range != output.etag:
ranges = None
if ranges is not None and not ranges:
return self.send_range_not_satisfiable(output.content_length)
for header in ('Accept-Ranges', 'Content-Encoding',
'Transfer-Encoding', 'ETag', 'Content-Length'):
outheaders.pop(header, all=True)
matched = '*' in none_match or (output.etag
and output.etag in none_match)
if matched:
if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag)
else:
self.simple_response(httplib.PRECONDITION_FAILED)
return
output.ranges = None
if output.etag and self.method in ('GET', 'HEAD'):
outheaders.set('ETag', output.etag, replace_all=True)
if accept_ranges:
outheaders.set('Accept-Ranges', 'bytes', replace_all=True)
if compressible and not ranges:
outheaders.set('Content-Encoding', 'gzip', replace_all=True)
if getattr(output, 'content_length', None):
outheaders.set('Calibre-Uncompressed-Length',
'%d' % output.content_length)
output = GeneratedOutput(compress_readable_output(output.src_file),
etag=output.etag)
if output.content_length is not None and not compressible and not ranges:
outheaders.set('Content-Length',
'%d' % output.content_length,
replace_all=True)
if compressible or output.content_length is None:
outheaders.set('Transfer-Encoding', 'chunked', replace_all=True)
if ranges:
if len(ranges) == 1:
r = ranges[0]
outheaders.set('Content-Length',
'%d' % r.size,
replace_all=True)
outheaders.set('Content-Range',
'bytes %d-%d/%d' %
(r.start, r.stop, output.content_length),
replace_all=True)
output.ranges = r
else:
range_parts = get_range_parts(ranges,
outheaders.get('Content-Type'),
output.content_length)
size = sum(map(len, range_parts)) + sum(r.size + 4
for r in ranges)
outheaders.set('Content-Length', '%d' % size, replace_all=True)
outheaders.set('Content-Type',
'multipart/byteranges; boundary=' +
MULTIPART_SEPARATOR,
replace_all=True)
output.ranges = zip_longest(ranges, range_parts)
request.status_code = httplib.PARTIAL_CONTENT
return output
def header_list_to_file(buf): # {{{
buf.append('')
return ReadOnlyFileBuffer(b''.join(
(x + '\r\n').encode('ascii') for x in buf))