def generate(self):
"""Generate and save the code for this snippet bundle."""
template = 'base/fetch_snippets.jinja'
if self.client.startpage_version == 5:
template = 'base/fetch_snippets_as.jinja'
bundle_content = render_to_string(template, {
'snippet_ids': [snippet.id for snippet in self.snippets],
'snippets_json': json.dumps([s.to_dict() for s in self.snippets]),
'client': self.client,
'locale': self.client.locale,
'settings': settings,
'current_firefox_major_version': util.current_firefox_major_version(),
})
if isinstance(bundle_content, str):
bundle_content = bundle_content.encode('utf-8')
if (settings.BUNDLE_BROTLI_COMPRESS and self.client.startpage_version >= 5):
content_file = ContentFile(brotli.compress(bundle_content))
content_file.content_encoding = 'br'
else:
content_file = ContentFile(bundle_content)
default_storage.save(self.filename, content_file)
cache.set(self.cache_key, True, ONE_DAY)
def brotli_compress(infile, level, decompress):
"""
@param infile
@param level
@param decompress
@return CompressionData
Compresses one file using the brotli algorithm by google.
"""
original_size = int(os.stat(infile).st_size)
with open(infile, "rU") as fdorig:
origlines = fdorig.read()
origtext = memoryview(bytearray(origlines, "utf8"))
# compressedtext = memoryview(zlib.compress(origtext.tobytes(), int(level)))
compressedtext = memoryview(brotli.compress(origtext.tobytes(), quality=int(level)))
compressed_size = len(compressedtext)
decompress_time = None
if decompress:
decompress_time = min(timeit.repeat(lambda: zlib.decompress(compressedtext.tobytes()),
number=10,
repeat=3, timer=time.clock))
cd = CompressionData(original_size, compressed_size, decompress_time)
return cd
def process_response(self, request, response):
if (response.streaming or
response.has_header('Content-Encoding') or
not self._accepts_brotli_encoding(request) or
len(response.content) < self.MIN_LEN_RESPONSE_TO_PROCESS):
# ---------
# 1) Skip streaming content, GZipMiddleware will compress it
# (supported, see https://github.com/google/brotli/issues/191).
# 2) Skip if the content is already encoded.
# 3) Skip if client didn't request brotli.
# 4) Skip if the content is short, compressing isn't worth it
# (same logic as Django's GZipMiddleware).
# ---------
return response
compressed_content = brotli.compress(response.content, quality=5)
# Return the uncompressed content if compression didn't help
if len(compressed_content) >= len(response.content):
return response
response.content = compressed_content
patch_vary_headers(response, ('Accept-Encoding',))
response['Content-Length'] = str(len(compressed_content))
response['Content-Encoding'] = 'br'
return response
def setUpClass(cls): assert os.path.exists(METADATA) with open(METADATA, 'rb') as f: cls.xml_metadata = f.read() cls.compressed_metadata = brotli.compress(cls.xml_metadata, mode=brotli.MODE_TEXT) # make random byte strings; font data must be 4-byte aligned cls.fontdata = bytes(bytearray(random.sample(range(0, 256), 80))) cls.privData = bytes(bytearray(random.sample(range(0, 256), 20)))
def test_http_payload_brotli(self, stream):
compressed = brotli.compress(b'brotli data')
out = aiohttp.FlowControlDataQueue(stream)
p = HttpPayloadParser(
out, length=len(compressed), compression='br')
p.feed_data(compressed)
assert b'brotli data' == b''.join(d for d, _ in out._buffer)
assert out.is_eof()
def test_roundtrip_compression_with_files(simple_compressed_file):
"""
Roundtripping data through the compressor works correctly.
"""
with open(simple_compressed_file[0], 'rb') as f:
uncompressed_data = f.read()
assert brotli.decompress(
brotli.compress(uncompressed_data)
) == uncompressed_data
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
print "{} wrote:".format(self.client_address[0])
print self.data
self.data = """HTTP/1.1 200 ok
Server: testbrotli
Date: Wed, 23 Sep 2015 05:52:16 GMT
Content-Type: text/plain
Connection: keep-alive
Content-Encoding: brotli
"""
self.data += brotli.compress("AAAAAAAA")
self.request.sendall(self.data)
def test_brotli_not_accepted_auto_decode(self):
import brotli
content = brotli.compress('ABCDEFG'.encode('utf-8'))
headers = {'Content-Type': 'application/octet-stream',
'Content-Encoding': 'br',
'Content-Length': str(len(content))
}
headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701mp_')
assert 'Content-Encoding' not in headers
assert 'Content-Length' not in headers
assert headers['X-Archive-Orig-Content-Encoding'] == 'br'
assert b''.join(gen).decode('utf-8') == 'ABCDEFG'
def test_brotli_accepted_no_change(self):
import brotli
content = brotli.compress('ABCDEFG'.encode('utf-8'))
headers = {'Content-Type': 'application/octet-stream',
'Content-Encoding': 'br',
'Content-Length': str(len(content))
}
headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701mp_',
environ={'HTTP_ACCEPT_ENCODING': 'gzip, deflate, br'})
assert headers['Content-Encoding'] == 'br'
assert headers['Content-Length'] == str(len(content))
assert brotli.decompress(b''.join(gen)).decode('utf-8') == 'ABCDEFG'
def close(self):
""" All tags must have been specified. Now write the table data and directory.
"""
if len(self.tables) != self.numTables:
raise TTLibError("wrong number of tables; expected %d, found %d" % (self.numTables, len(self.tables)))
if self.sfntVersion in ("\x00\x01\x00\x00", "true"):
isTrueType = True
elif self.sfntVersion == "OTTO":
isTrueType = False
else:
raise TTLibError("Not a TrueType or OpenType font (bad sfntVersion)")
# The WOFF2 spec no longer requires the glyph offsets to be 4-byte aligned.
# However, the reference WOFF2 implementation still fails to reconstruct
# 'unpadded' glyf tables, therefore we need to 'normalise' them.
# See:
# https://github.com/khaledhosny/ots/issues/60
# https://github.com/google/woff2/issues/15
if isTrueType:
self._normaliseGlyfAndLoca(padding=4)
self._setHeadTransformFlag()
# To pass the legacy OpenType Sanitiser currently included in browsers,
# we must sort the table directory and data alphabetically by tag.
# See:
# https://github.com/google/woff2/pull/3
# https://lists.w3.org/Archives/Public/public-webfonts-wg/2015Mar/0000.html
# TODO(user): remove to match spec once browsers are on newer OTS
self.tables = OrderedDict(sorted(self.tables.items()))
self.totalSfntSize = self._calcSFNTChecksumsLengthsAndOffsets()
fontData = self._transformTables()
compressedFont = brotli.compress(fontData, mode=brotli.MODE_FONT)
self.totalCompressedSize = len(compressedFont)
self.length = self._calcTotalSize()
self.majorVersion, self.minorVersion = self._getVersion()
self.reserved = 0
directory = self._packTableDirectory()
self.file.seek(0)
self.file.write(pad(directory + compressedFont, size=4))
self._writeFlavorData()
def brotli(f, *args, **kwargs):
"""Brotli Flask Response Decorator"""
data = f(*args, **kwargs)
if isinstance(data, Response):
content = data.data
else:
content = data
deflated_data = _brotli.compress(content)
if isinstance(data, Response):
data.data = deflated_data
data.headers['Content-Encoding'] = 'br'
data.headers['Content-Length'] = str(len(data.data))
return data
return deflated_data
def getSFNTData(pathOrFile, unsortGlyfLoca=False, glyphBBox="", alt255UInt16=False):
font = TTFont(pathOrFile)
tableChecksums = {}
tableData = {}
tableOrder = [i for i in sorted(font.keys()) if len(i) == 4]
if unsortGlyfLoca:
assert "loca" in tableOrder
loca = tableOrder.index("loca")
glyf = tableOrder.index("glyf")
tableOrder.insert(glyf, tableOrder.pop(loca))
for tag in tableOrder:
tableChecksums[tag] = font.reader.tables[tag].checkSum
tableData[tag] = transformTable(font, tag, glyphBBox=glyphBBox, alt255UInt16=alt255UInt16)
totalData = "".join([tableData[tag][1] for tag in tableOrder])
compData = brotli.compress(totalData, brotli.MODE_FONT)
if len(compData) >= len(totalData):
compData = totalData
font.close()
del font
return tableData, compData, tableOrder, tableChecksums
def compress(data, level=9, method='gz'):
"""
Compress *data* with the specified *method*. It can be ``'zip'``,
``'gz'`` or ``'bz'``. If the :mod:`brotli` module is installed,
``'brt'`` is also an accepted method.
:raise ValueError: If an invalid *method* was supplied.
"""
if method == 'gz':
return gzip.compress(data, level)
elif method == 'zip':
return zlib.compress(data, level)
elif method == 'bz':
return bz2.compress(data, level)
elif method == 'brt':
if not brotli:
raise ImportError('brotli')
return brotli.compress(data)
else:
raise ValueError('invalid method: {0!r}'.format(method))
def main():
"""Testing 'brotlipy' package"""
parser = argparse.ArgumentParser()
parser.add_argument(
"-v",
"--verbosity",
dest="verbosity",
action="count",
default=0,
help="set verbosity level",
)
args = parser.parse_args()
if args.verbosity == 1:
logging.basicConfig(level=logging.INFO)
elif args.verbosity > 1:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.ERROR)
data = brotli.compress(b"test")
data = brotli.decompress(data)
def post_build(self, pkt, pay):
if not conf.contribs["http"]["auto_compression"]:
return pkt + pay
encodings = self._get_encodings()
# Compress
if "deflate" in encodings:
import zlib
pay = zlib.compress(pay)
elif "gzip" in encodings:
pay = gzip_compress(pay)
elif "compress" in encodings:
import lzw
pay = lzw.compress(pay)
elif "br" in encodings:
if _is_brotli_available:
pay = brotli.compress(pay)
else:
log_loading.info(
"Can't import brotli. brotli compression will "
"be ignored !"
)
return pkt + pay
async def get_many(self, request):
service = self.di()
page = request.query.getone('page', None)
elements = request.query.getone('elements', None)
url = str(request.url)
cached = await redis_pool.get(url)
if cached:
decompressed_bytes = brotli.decompress(cached)
response = web.Response(body=decompressed_bytes.decode('utf-8'),
content_type='application/json')
response.enable_compression()
return response
data = json.dumps({'data': await service.all(page, elements)})
compressed_data = brotli.compress(data.encode('utf-8'))
# await redis_pool.set(url, codecs.encode(data.encode('utf-8'), 'gzip'), expire=86400)
await redis_pool.set(url, compressed_data, expire=86400)
response = web.Response(body=data, content_type='application/json')
response.enable_compression()
return response
def _calcFlavorDataOffsetsAndSize(self, start): """Calculate offsets and lengths for any meta- and/or private data.""" offset = start data = self.flavorData if data.metaData: self.metaOrigLength = len(data.metaData) self.metaOffset = offset self.compressedMetaData = brotli.compress( data.metaData, mode=brotli.MODE_TEXT) self.metaLength = len(self.compressedMetaData) offset += self.metaLength else: self.metaOffset = self.metaLength = self.metaOrigLength = 0 self.compressedMetaData = b"" if data.privData: # make sure private data is padded to 4-byte boundary offset = (offset + 3) & ~3 self.privOffset = offset self.privLength = len(data.privData) offset += self.privLength else: self.privOffset = self.privLength = 0 return offset
def generate(self):
"""Generate and save the code for this snippet bundle."""
# Generate the new AS Router bundle format
data = [snippet.render() for snippet in self.snippets]
bundle_content = json.dumps({
'messages': data,
'metadata': {
'generated_at': datetime.utcnow().isoformat(),
'number_of_snippets': len(data),
}
})
if isinstance(bundle_content, str):
bundle_content = bundle_content.encode('utf-8')
if settings.BUNDLE_BROTLI_COMPRESS:
content_file = ContentFile(brotli.compress(bundle_content))
content_file.content_encoding = 'br'
else:
content_file = ContentFile(bundle_content)
default_storage.save(self.filename, content_file)
cache.set(self.cache_key, True, ONE_DAY)
def generate(self):
"""Generate and save the code for this snippet bundle."""
# Generate the new AS Router bundle format
data = [snippet.render() for snippet in self.snippets]
bundle_content = json.dumps({
'messages': data,
'metadata': {
'generated_at': datetime.utcnow().isoformat(),
'number_of_snippets': len(data),
}
})
if isinstance(bundle_content, str):
bundle_content = bundle_content.encode('utf-8')
if settings.BUNDLE_BROTLI_COMPRESS:
content_file = ContentFile(brotli.compress(bundle_content))
content_file.content_encoding = 'br'
else:
content_file = ContentFile(bundle_content)
default_storage.save(self.filename, content_file)
cache.set(self.cache_key, True, ONE_DAY)
def _calcFlavorDataOffsetsAndSize(self, start):
"""Calculate offsets and lengths for any meta- and/or private data."""
offset = start
data = self.flavorData
if data.metaData:
self.metaOrigLength = len(data.metaData)
self.metaOffset = offset
self.compressedMetaData = brotli.compress(data.metaData,
mode=brotli.MODE_TEXT)
self.metaLength = len(self.compressedMetaData)
offset += self.metaLength
else:
self.metaOffset = self.metaLength = self.metaOrigLength = 0
self.compressedMetaData = b""
if data.privData:
# make sure private data is padded to 4-byte boundary
offset = (offset + 3) & ~3
self.privOffset = offset
self.privLength = len(data.privData)
offset += self.privLength
else:
self.privOffset = self.privLength = 0
return offset
def encode_content_body(self, text, encoding):
logger.dbg('Encoding content to {}'.format(encoding))
data = text
if encoding == 'identity':
pass
elif encoding in ('gzip', 'x-gzip'):
_io = BytesIO()
with gzip.GzipFile(fileobj=_io, mode='wb') as f:
f.write(text)
data = _io.getvalue()
elif encoding == 'deflate':
data = zlib.compress(text)
elif encoding == 'br':
# Brotli algorithm
try:
data = brotli.compress(text)
except Exception as e:
#raise Exception('Could not compress Brotli stream: "{}"'.format(str(e)))
logger.err('Could not compress Brotli stream: "{}"'.format(str(e)))
else:
#raise Exception("Unknown Content-Encoding: %s" % encoding)
logger.err('Unknown Content-Encoding: "{}"'.format(encoding))
return data
def main(args):
options = parse_options(args)
if options.infile:
if not os.path.isfile(options.infile):
print('file "%s" not found' % options.infile, file=sys.stderr)
sys.exit(1)
with open(options.infile, "rb") as infile:
data = infile.read()
else:
if sys.stdin.isatty():
# interactive console, just quit
usage()
infile = get_binary_stdio('stdin')
data = infile.read()
if options.outfile:
if os.path.isfile(options.outfile) and not options.force:
print('output file exists')
sys.exit(1)
outfile = open(options.outfile, "wb")
else:
outfile = get_binary_stdio('stdout')
try:
if options.decompress:
data = brotli.decompress(data)
else:
data = brotli.compress(data, options.mode, options.transform)
except brotli.error as e:
print('[ERROR] %s: %s' % (e, options.infile or 'sys.stdin'),
file=sys.stderr)
sys.exit(1)
outfile.write(data)
outfile.close()
def main(args):
options = parse_options(args)
if options.infile:
if not os.path.isfile(options.infile):
print('file "%s" not found' % options.infile, file=sys.stderr)
sys.exit(1)
with open(options.infile, "rb") as infile:
data = infile.read()
else:
if sys.stdin.isatty():
# interactive console, just quit
usage()
infile = get_binary_stdio('stdin')
data = infile.read()
if options.outfile:
if os.path.isfile(options.outfile) and not options.force:
print('output file exists')
sys.exit(1)
outfile = open(options.outfile, "wb")
else:
outfile = get_binary_stdio('stdout')
try:
if options.decompress:
data = brotli.decompress(data)
else:
data = brotli.compress(data, options.mode, options.transform)
except brotli.error as e:
print('[ERROR] %s: %s' % (e, options.infile or 'sys.stdin'),
file=sys.stderr)
sys.exit(1)
outfile.write(data)
outfile.close()
async def format_download_form(file_path: Path, base_path: Path, dir_path: Path, compress: bool, rename: str = None) \
-> FormData:
if compress and get_content_encoding(file_path) == "br":
path_log = "/".join(file_path.parts[2:])
start = time.time()
if which('brotli'):
log_info(f'brotlify (system) {path_log} ...')
p = await asyncio.create_subprocess_shell(
cmd=f'brotli {str(file_path)}',
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
shell=True)
async for f in p.stderr:
log_error(f.decode('utf-8'))
await p.communicate()
os.system(f'rm {str(file_path)}')
os.system(f'mv {str(file_path)}.br {str(file_path)}')
else:
log_info(f'brotlify (python) {path_log}')
start = time.time()
compressed = brotli.compress(file_path.read_bytes())
with file_path.open("wb") as f:
f.write(compressed)
log_info(f'...{path_log} => {time.time() - start} s')
data = open(str(file_path), 'rb').read()
path_bucket = base_path / file_path.relative_to(
dir_path) if not rename else base_path / rename
return FormData(objectName=path_bucket,
objectData=data,
owner=Configuration.owner,
objectSize=len(data),
content_type=get_content_type(file_path.name),
content_encoding=get_content_encoding(file_path.name))
def brotli(self,
INPUT_FILE='system.new.dat.br',
OUTPUT_FILE='system.new.dat',
flag=1):
import brotli as b
if quiet == 0:
if input('是否转换.new.dat.br?y/n>>>') == 'n': return
if flag == 1:
with open(INPUT_FILE, 'rb') as infile:
data = infile.read()
outfile = open(OUTPUT_FILE, 'wb')
data = b.decompress(data)
outfile.write(data)
outfile.close()
infile.close()
if flag == 2:
with open(INPUT_FILE, 'rb') as infile:
data = infile.read()
outfile = open(OUTPUT_FILE, 'wb')
data = b.compress(data)
outfile.write(data)
outfile.close()
infile.close()
print('参数无效!')
def compressedFiles(path):
for filepath in shipItFiles(path):
try:
filecontent = pathlib.Path(filepath).read_bytes()
except FileNotFoundError:
print("Skip symlink or unreadable {0}".format(filepath))
continue
if (args.compress_gzip):
compressedContent = gzip.compress(filecontent)
yield filepath, compressedContent
continue
if (args.compress_brotli):
try:
import brotli
except ImportError:
print("Error: Brotli compressor is not installed, run pip(3) install brotli")
exit(1)
compressedContent = brotli.compress(filecontent)
yield filepath, compressedContent
continue
yield filepath, filecontent
def getSFNTData(pathOrFile, unsortGlyfLoca=False, glyphBBox="", alt255UInt16=False):
if isinstance(pathOrFile, TTFont):
font = pathOrFile
else:
font = getTTFont(pathOrFile)
tableChecksums = {}
tableData = {}
tableOrder = [i for i in sorted(font.keys()) if len(i) == 4]
if unsortGlyfLoca:
assert "loca" in tableOrder
loca = tableOrder.index("loca")
glyf = tableOrder.index("glyf")
tableOrder.insert(glyf, tableOrder.pop(loca))
for tag in tableOrder:
tableChecksums[tag] = font.reader.tables[tag].checkSum
tableData[tag] = transformTable(font, tag, glyphBBox=glyphBBox, alt255UInt16=alt255UInt16)
totalData = "".join([tableData[tag][1] for tag in tableOrder])
compData = brotli.compress(totalData, brotli.MODE_FONT)
if len(compData) >= len(totalData):
compData = totalData
if not isinstance(pathOrFile, TTFont):
font.close()
del font
return tableData, compData, tableOrder, tableChecksums
def brotli_compress(content, quality=None):
if quality is None:
# 5/6 are good balance between compression speed and compression rate
quality = 5
return brotli.compress(content, quality=quality)
def test_garbage_appended(self):
with self.assertRaises(brotli.error):
self.decompressor.process(brotli.compress(b'a') + b'a')
def handle(self, *args, **options):
if not options['timestamp']:
self.stdout.write('Generating all bundles.')
total_jobs = Job.objects.all()
else:
self.stdout.write(
'Generating bundles with Jobs modified on or after {}'.format(options['timestamp'])
)
total_jobs = Job.objects.filter(
Q(snippet__modified__gte=options['timestamp']) |
Q(distribution__distributionbundle__modified__gte=options['timestamp'])
).distinct()
if not total_jobs:
self.stdout.write('Nothing to do…')
return
self.stdout.write('Processing bundles…')
combinations_to_process = set(
itertools.chain.from_iterable(
itertools.product(
job.channels,
job.snippet.locale.code.strip(',').split(',')
)
for job in total_jobs
)
)
distribution_bundles_to_process = DistributionBundle.objects.filter(
distributions__jobs__in=total_jobs
).distinct().order_by('id')
for distribution_bundle in distribution_bundles_to_process:
distributions = distribution_bundle.distributions.all()
for channel, locale in combinations_to_process:
additional_jobs = []
if channel == 'nightly' and settings.NIGHTLY_INCLUDES_RELEASE:
additional_jobs = Job.objects.filter(
status=Job.PUBLISHED).filter(**{
'targets__on_release': True,
'distribution__in': distributions,
})
channel_jobs = Job.objects.filter(
status=Job.PUBLISHED).filter(
Q(**{
'targets__on_{}'.format(channel): True,
'distribution__in': distributions,
}))
all_jobs = Job.objects.filter(
Q(id__in=additional_jobs) | Q(id__in=channel_jobs)
)
locales_to_process = [
key.lower() for key in product_details.languages.keys()
if key.lower().startswith(locale)
]
for locale_to_process in locales_to_process:
filename = 'Firefox/{channel}/{locale}/{distribution}.json'.format(
channel=channel,
locale=locale_to_process,
distribution=distribution_bundle.code_name,
)
filename = os.path.join(settings.MEDIA_BUNDLES_PREGEN_ROOT, filename)
full_locale = ',{},'.format(locale_to_process.lower())
splitted_locale = ',{},'.format(locale_to_process.lower().split('-', 1)[0])
bundle_jobs = all_jobs.filter(
Q(snippet__locale__code__contains=splitted_locale) |
Q(snippet__locale__code__contains=full_locale)).distinct()
# If DistributionBundle is not enabled, or if there are no
# Published Jobs for the channel / locale / distribution
# combination, delete the current bundle file if it exists.
if not distribution_bundle.enabled or not bundle_jobs.exists():
if default_storage.exists(filename):
self.stdout.write('Removing {}'.format(filename))
default_storage.delete(filename)
continue
data = []
channel_job_ids = list(channel_jobs.values_list('id', flat=True))
for job in bundle_jobs:
if job.id in channel_job_ids:
render = job.render()
else:
render = job.render(always_eval_to_false=True)
data.append(render)
bundle_content = json.dumps({
'messages': data,
'metadata': {
'generated_at': datetime.utcnow().isoformat(),
'number_of_snippets': len(data),
'channel': channel,
}
})
# Convert str to bytes.
if isinstance(bundle_content, str):
bundle_content = bundle_content.encode('utf-8')
if settings.BUNDLE_BROTLI_COMPRESS:
content_file = ContentFile(brotli.compress(bundle_content))
content_file.content_encoding = 'br'
else:
content_file = ContentFile(bundle_content)
default_storage.save(filename, content_file)
self.stdout.write(self.style.SUCCESS('Writing bundle {}'.format(filename)))
#!/usr/bin/env python
import sys
import zlib
import lz4
import brotli
from pcapfile import savefile
for arg in sys.argv[1:]:
pcap = savefile.load_savefile(open(arg, 'r'))
for i in range(0, len(pcap.packets)):
# strip IP/UDP
pkt = pcap.packets[i].raw()[42:]
# do not compress header
zlib_len = len(zlib.compress(pkt[12:])) + 12
lz4_len = len(lz4.compress(pkt[12:])) + 12
bro_len = len(brotli.compress(pkt[12:])) + 12
print('[%d] base: %4dB, zlib: %4dB, lz4: %4dB, brotli: %4dB' % \
(i, len(pkt), zlib_len, lz4_len, bro_len))
def _brotli_compress(self, original_file):
original_file.seek(0)
content = BytesIO(brotli.compress(original_file.read()))
content.seek(0)
return File(content)
def main(args=None):
parser = argparse.ArgumentParser(
prog='bro.py',
description=
"Compression/decompression utility using the Brotli algorithm.")
parser.add_argument('--version',
action='version',
version=brotli.__version__)
parser.add_argument('-i',
'--input',
metavar='FILE',
type=str,
dest='infile',
help='Input file',
default=None)
parser.add_argument('-o',
'--output',
metavar='FILE',
type=str,
dest='outfile',
help='Output file',
default=None)
parser.add_argument('-f',
'--force',
action='store_true',
help='Overwrite existing output file',
default=False)
parser.add_argument('-d',
'--decompress',
action='store_true',
help='Decompress input file',
default=False)
params = parser.add_argument_group('optional encoder parameters')
params.add_argument(
'-m',
'--mode',
metavar="MODE",
type=int,
choices=[0, 1],
help='The compression mode can be 0 for generic input, '
'1 for UTF-8 encoded text, or 2 for WOFF 2.0 font data. '
'Defaults to 0.')
params.add_argument(
'-q',
'--quality',
metavar="QUALITY",
type=int,
choices=list(range(0, 12)),
help='Controls the compression-speed vs compression-density '
'tradeoff. The higher the quality, the slower the '
'compression. Range is 0 to 11. Defaults to 11.')
params.add_argument(
'--lgwin',
metavar="LGWIN",
type=int,
choices=list(range(16, 25)),
help='Base 2 logarithm of the sliding window size. Range is '
'10 to 24. Defaults to 22.')
params.add_argument(
'--lgblock',
metavar="LGBLOCK",
type=int,
choices=[0] + list(range(16, 25)),
help='Base 2 logarithm of the maximum input block size. '
'Range is 16 to 24. If set to 0, the value will be set based '
'on the quality. Defaults to 0.')
# set default values using global DEFAULT_PARAMS dictionary
parser.set_defaults(**DEFAULT_PARAMS)
options = parser.parse_args(args=args)
if options.infile:
if not os.path.isfile(options.infile):
parser.error('file "%s" not found' % options.infile)
with open(options.infile, "rb") as infile:
data = infile.read()
else:
if sys.stdin.isatty():
# interactive console, just quit
parser.error('no input')
infile = get_binary_stdio('stdin')
data = infile.read()
if options.outfile:
if os.path.isfile(options.outfile) and not options.force:
parser.error('output file exists')
outfile = open(options.outfile, "wb")
else:
outfile = get_binary_stdio('stdout')
try:
if options.decompress:
data = brotli.decompress(data)
else:
data = brotli.compress(data,
mode=options.mode,
quality=options.quality,
lgwin=options.lgwin,
lgblock=options.lgblock)
except brotli.error as e:
parser.exit(1,
'bro: error: %s: %s' % (e, options.infile or 'sys.stdin'))
outfile.write(data)
outfile.close()
def _encode_payload(cls, payload: dict) -> str:
text = json.dumps(payload, ensure_ascii=False)
data = brotli.compress(text.encode('utf-8'), quality=6)
return cls._b64encode(data)
def encode_brotli(content: bytes) -> bytes:
return brotli.compress(content)
def encode_brotli(content):
return brotli.compress(content)
with open('/home/folkertdev/elm/elm-brotli/tests/Generated.elm', 'w') as output:
output.write(file_format.format(contents, result))
if __name__ == '__main__':
bases = []
for sourcePath in glob.glob("sources/**"):
bases.append(os.path.splitext(os.path.basename(sourcePath))[0])
for sourcePath in glob.glob("sources/**"):
with open(sourcePath) as source:
contents = source.read()
bs = contents.encode('utf-8')
bs = brotli.compress(bs)
result = [ v for v in bs ]
base = os.path.basename(sourcePath)
destName = os.path.splitext(base)[0]
with open(destName + ".elm", 'w') as dst:
dst.write(test_file_format.format(destName, contents, result))
imports = "\n".join("import Gutenberg." + name + " as " + name for name in bases)
tests = ", ".join("pipeline \"" + name + "\" " + name + ".text " + name + ".bytes" for name in bases)
with open("../Gutenberg.elm", 'w') as dst:
dst.write(gutenberg_format.format(imports, tests))
def _compress(self, test_data, **kwargs):
temp_compressed = _test_utils.get_temp_compressed_name(test_data)
with open(temp_compressed, 'wb') as out_file:
with open(test_data, 'rb') as in_file:
out_file.write(brotli.compress(in_file.read(), **kwargs))
def defaultTestData(header=None, directory=None, collectionHeader=None, collectionDirectory=None, tableData=None, compressedData=None, metadata=None, privateData=None, flavor="cff", Base128Bug=False):
isCollection = collectionDirectory is not None
parts = []
# setup the header
if header is None:
header = deepcopy(testDataWOFFHeader)
parts.append(header)
# setup the directory
if directory is None:
if flavor == "cff":
directory = deepcopy(testCFFDataWOFFDirectory)
else:
directory = deepcopy(testTTFDataWOFFDirectory)
parts.append(directory)
if isCollection:
if collectionHeader is None:
collectionHeader = dict(version=0x00010000, numFonts=len(collectionDirectory))
parts.append(collectionHeader)
parts.append(collectionDirectory)
# setup the table data
if tableData is None:
if flavor == "cff":
tableData = deepcopy(sfntCFFTableData)
else:
tableData = deepcopy(sfntTTFTableData)
if compressedData is None:
if flavor == "cff":
compressedData = deepcopy(sfntCFFCompressedData)
else:
compressedData = deepcopy(sfntTTFCompressedData)
parts.append(compressedData)
# sanity checks
assert len(directory) == len(tableData)
if not isCollection:
assert set(tableData.keys()) == set([entry["tag"] for entry in directory])
# apply the directory data to the header
header["numTables"] = len(directory)
if isCollection:
header["flavor"] = "ttcf"
elif "CFF " in tableData:
header["flavor"] = "OTTO"
else:
header["flavor"] = "\000\001\000\000"
# apply the table data to the directory and the header
if isCollection:
# TTC header
header["totalSfntSize"] = 12 + 4 * collectionHeader["numFonts"]
header["totalSfntSize"] += sfntDirectorySize * collectionHeader["numFonts"]
for entry in collectionDirectory:
header["totalSfntSize"] += sfntDirectoryEntrySize * entry["numTables"]
else:
header["totalSfntSize"] = sfntDirectorySize + (len(directory) * sfntDirectoryEntrySize)
header["totalCompressedSize"] = len(compressedData)
for i, entry in enumerate(directory):
tag = entry["tag"]
if isCollection:
origData, transformData = tableData[i][1]
else:
origData, transformData = tableData[tag]
entry["origLength"] = len(origData)
entry["transformLength"] = len(transformData)
if tag == "hmtx" and entry["origLength"] > entry["transformLength"]:
entry["transformFlag"] = 1
header["totalSfntSize"] += entry["origLength"]
header["totalSfntSize"] += calcPaddingLength(header["totalSfntSize"])
header["length"] = woffHeaderSize + len(packTestDirectory(directory, Base128Bug=Base128Bug))
if isCollection:
header["length"] += len(packTestCollectionHeader(collectionHeader))
header["length"] += len(packTestCollectionDirectory(collectionDirectory))
header["length"] += len(compressedData)
header["length"] += calcPaddingLength(header["length"])
# setup the metadata
if metadata is not None:
if isinstance(metadata, tuple):
metadata, compMetadata = metadata
else:
compMetadata = None
if compMetadata is None:
compMetadata = brotli.compress(metadata, brotli.MODE_TEXT)
header["metaOffset"] = header["length"]
header["metaLength"] = len(compMetadata)
header["metaOrigLength"] = len(metadata)
header["length"] += len(compMetadata)
if privateData is not None:
header["length"] += calcPaddingLength(len(compMetadata))
parts.append((metadata, compMetadata))
# setup the private data
if privateData is not None:
header["privOffset"] = header["length"]
header["privLength"] = len(privateData)
header["length"] += len(privateData)
parts.append(privateData)
# return the parts
return parts
def test_already_finished(self):
self.decompressor.process(brotli.compress(b'a'))
with self.assertRaises(brotli.error):
self.decompressor.process(b'a')
def main(args=None):
parser = argparse.ArgumentParser(
prog='bro.py',
description="Compression/decompression utility using the Brotli algorithm.")
parser.add_argument('--version', action='version', version=brotli.__version__)
parser.add_argument('-i', '--input', metavar='FILE', type=str, dest='infile',
help='Input file', default=None)
parser.add_argument('-o', '--output', metavar='FILE', type=str, dest='outfile',
help='Output file', default=None)
parser.add_argument('-f', '--force', action='store_true',
help='Overwrite existing output file', default=False)
parser.add_argument('-d', '--decompress', action='store_true',
help='Decompress input file', default=False)
params = parser.add_argument_group('optional encoder parameters')
params.add_argument('-m', '--mode', metavar="MODE", type=int, choices=[0, 1],
help='The compression mode can be 0 for generic input, '
'1 for UTF-8 encoded text, or 2 for WOFF 2.0 font data. '
'Defaults to 0.')
params.add_argument('-q', '--quality', metavar="QUALITY", type=int,
choices=list(range(0, 12)),
help='Controls the compression-speed vs compression-density '
'tradeoff. The higher the quality, the slower the '
'compression. Range is 0 to 11. Defaults to 11.')
params.add_argument('--lgwin', metavar="LGWIN", type=int,
choices=list(range(16, 25)),
help='Base 2 logarithm of the sliding window size. Range is '
'10 to 24. Defaults to 22.')
params.add_argument('--lgblock', metavar="LGBLOCK", type=int,
choices=[0] + list(range(16, 25)),
help='Base 2 logarithm of the maximum input block size. '
'Range is 16 to 24. If set to 0, the value will be set based '
'on the quality. Defaults to 0.')
# set default values using global DEFAULT_PARAMS dictionary
parser.set_defaults(**DEFAULT_PARAMS)
options = parser.parse_args(args=args)
if options.infile:
if not os.path.isfile(options.infile):
parser.error('file "%s" not found' % options.infile)
with open(options.infile, "rb") as infile:
data = infile.read()
else:
if sys.stdin.isatty():
# interactive console, just quit
parser.error('no input')
infile = get_binary_stdio('stdin')
data = infile.read()
if options.outfile:
if os.path.isfile(options.outfile) and not options.force:
parser.error('output file exists')
outfile = open(options.outfile, "wb")
else:
outfile = get_binary_stdio('stdout')
try:
if options.decompress:
data = brotli.decompress(data)
else:
data = brotli.compress(
data, mode=options.mode, quality=options.quality,
lgwin=options.lgwin, lgblock=options.lgblock)
except brotli.error as e:
parser.exit(1,'bro: error: %s: %s' % (e, options.infile or 'sys.stdin'))
outfile.write(data)
outfile.close()
async def ReplyCompressBrotli(self, connection, header):
header["Additional"].append(b"Content-Encoding: br")
tmp_compress = brotli.compress(header["ReplyContent"])
header["ReplyContent"] = tmp_compress
await self.Reply(connection, header)
def handle(self, *args, **options):
self.stdout.write("Generating JSON...")
now = time.time()
try:
os.mkdir("json")
except FileExistsError:
# Folder already exists
pass
# Write new files next to the old ones, then atomically replace
with open("json/coords.geojson.tmp", "w", encoding="utf-8") as f:
records = (SchoolV2.objects.exclude(lat__isnull=True).exclude(
lon__isnull=True).exclude(lat=0,
lon=0).values("id", "lat",
"lon").iterator())
features = [{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [record["lon"], record["lat"]],
},
"properties": {
"id": record["id"]
},
} for record in records]
count = len(features)
feature_collection = {
"type": "FeatureCollection",
"features": features
}
json.dump(feature_collection, f, separators=(",", ":"))
self.stdout.write(
self.style.SUCCESS(
f"Wrote {count} records in {round(time.time()-now, 3)} seconds"
))
# Generate compressed versions
with open("json/coords.geojson.tmp", "rb") as f:
data = f.read()
# gzip
self.stdout.write("Compressing with gzip...")
now = time.time()
with gzip.open("json/coords.geojson.gz.tmp", "wb") as g:
g.write(data)
self.stdout.write(
self.style.SUCCESS(
f"Finished in {round(time.time()-now, 3)} seconds"))
# brotli takes a long time, don't bother when testing
if not settings.DEBUG:
self.stdout.write("Compressing with brotli...")
now = time.time()
with open("json/coords.geojson.br.tmp", "wb") as b:
b.write(brotli.compress(data, mode=brotli.MODE_TEXT))
self.stdout.write(
self.style.SUCCESS(
f"Finished in {round(time.time()-now, 3)} seconds"))
else:
self.stdout.write("Skipping brotli...")
# Replace
os.replace("json/coords.geojson.tmp", "json/coords.geojson")
os.replace("json/coords.geojson.gz.tmp", "json/coords.geojson.gz")
if not settings.DEBUG:
os.replace("json/coords.geojson.br.tmp", "json/coords.geojson.br")
def setUpFlavorData(cls): assert os.path.exists(METADATA) with open(METADATA, 'rb') as f: cls.xml_metadata = f.read() cls.compressed_metadata = brotli.compress(cls.xml_metadata, mode=brotli.MODE_TEXT) cls.privData = bytes(bytearray(random.sample(range(0, 256), 20)))
import brotli
import random
import string
with open('./silesia.tar', 'rb') as f:
decoded = f.read()
# with open('ipsum.brotli', 'wb') as f:
with open('silesia-5.brotli', 'wb') as f:
compressed = brotli.compress(decoded, quality=5)
f.write(compressed)
def brotli_encode(data, level=11, mode=0, lgwin=22, out=None):
"""Compress Brotli."""
return brotli.compress(data, quality=level, mode=mode, lgwin=lgwin)
def update_geojson_summary(args, stations, updated_stations, summary):
stations_with_ascents = dict()
# unroll into dicts for quick access
if 'features' in summary:
for feature in summary.features:
a = feature.properties['ascents']
if len(a):
st_id = a[0]['station_id']
stations_with_ascents[st_id] = feature
# remove entries from ascents which have a syn_timestamp less than cutoff_ts
cutoff_ts = now() - args.max_age
# now walk the updates
for id, asc in updated_stations:
if id in stations_with_ascents:
# we already have ascents from this station.
# append, sort by synoptic time and de-duplicate
oldlist = stations_with_ascents[id]['properties']['ascents']
oldlist.append(asc)
pruned = [x for x in oldlist if x['syn_timestamp'] > cutoff_ts]
logging.debug(f"pruning {id}: {len(oldlist)} -> {len(pruned)}")
newlist = sorted(pruned,
key=itemgetter('syn_timestamp'),
reverse=True)
# https://stackoverflow.com/questions/9427163/remove-duplicate-dict-in-list-in-python
seen = set()
dedup = []
for d in newlist:
# keep an ascent of each source, even if same synop time
t = str(d['syn_timestamp']) + d['source']
if t not in seen:
seen.add(t)
dedup.append(d)
stations_with_ascents[id]['properties']['ascents'] = dedup
else:
# station appears with first-time ascent
properties = dict()
properties["ascents"] = [asc]
if id in stations:
st = stations[id]
coords = (st['lon'], st['lat'], st['elevation'])
properties["name"] = st['name']
else:
# unlisted station: anonymous + mobile
# take coords and station_id as name from ascent
coords = (asc['lon'], asc['lat'], asc['elevation'])
properties["name"] = asc['station_id']
stations_with_ascents[id] = geojson.Feature(
geometry=geojson.Point(coords), properties=properties)
# create GeoJSON summary
ns = na = 0
fc = geojson.FeatureCollection([])
for st, f in stations_with_ascents.items():
ns += 1
na += len(f.properties['ascents'])
fc.features.append(f)
gj = geojson.dumps(fc, indent=4)
dest = os.path.splitext(args.summary)[0]
if not dest.endswith(".br"):
dest += '.br'
logging.debug(f"summary {dest}: {ns} active stations, {na} ascents")
fd, path = tempfile.mkstemp(dir=args.tmpdir)
src = gj.encode("utf8")
start = time.time()
dst = brotli.compress(src, quality=BROTLI_SUMMARY_QUALITY)
end = time.time()
dt = end - start
sl = len(src)
dl = len(dst)
ratio = (1. - dl / sl) * 100.
logging.debug(
f"summary {dest}: brotli {sl} -> {dl}, compression={ratio:.1f}% in {dt:.3f}s"
)
os.write(fd, dst)
os.fsync(fd)
os.close(fd)
os.rename(path, dest)
os.chmod(dest, 0o644)
def brotli_file(filepath):
destination = filepath + ".br"
with open(filepath, "rb") as source, open(destination, "wb") as dest:
dest.write(brotli.compress(source.read()))
return destination
def compress(data: str):
c = brotli.compress(bytes(data, encoding='utf-8'))
return base64.b64encode(c).decode('utf-8')
def compress_with_brotli(data: bytes) -> bytes:
"""Compress binary data using brotli compression."""
return brotli.compress(data, mode=brotli.MODE_TEXT, quality=11)
def test_value_error_without_request(header_value):
headers = [(b"Content-Encoding", header_value)]
body = b"test 123"
compressed_body = brotli.compress(body)[3:]
with pytest.raises(ValueError):
httpx.Response(200, headers=headers, content=compressed_body)
def compress(self, path, file):
return ContentFile(brotli.compress(file.read()))
def do_compress(self, filename, compressed_filename):
import brotli
with open(filename, 'rb') as f_in, \
open(compressed_filename, 'wb') as f_out:
f_out.write(brotli.compress(f_in.read(), mode=brotli.MODE_TEXT))
def brotli_compress(content):
return compress(content, quality=DEFAULT_LEVEL)
def compress_brotli(data):
return brotli.compress(data)
def main(truthVersion: str = None):
if not truthVersion:
truthVersion = input("TruthVersion: ")
# Download the database if TruthVersion is exist
r = requests.get(
f"https://img-pc.so-net.tw/dl/Resources/{truthVersion}/Jpn/AssetBundles/Android/manifest/masterdata_assetmanifest",
headers=header,
)
if r.status_code != 200:
print(f"TruthVersion {truthVersion} is not exist")
return
print(f"TruthVersion {truthVersion} is exist")
filename, path, _, size, _ = r.text.split(",")
print(f"Downloading asset bundle ...")
r = requests.get(
f"https://img-pc.so-net.tw/dl/pool/AssetBundles/{path[:2]}/{path}",
headers=header)
if r.headers.get("Content-Length") != size:
print("Size is not same, but it may be fine")
with open(os.path.join(script_dir, "masterdata_master.unity3d"),
"wb+") as f:
f.write(r.content)
masterDB = None
# Unpack asset bundle
with open("masterdata_master.unity3d", "rb") as f:
bundle = unitypack.load(f)
for asset in bundle.assets:
for id, object in asset.objects.items():
if object.type == "TextAsset":
data = object.read()
masterDB = data.script
break
os.remove(os.path.join(script_dir, "masterdata_master.unity3d"))
# Compress
print("Compressing redive_tw.db.br ...")
brotliDB = brotli.compress(masterDB)
# Hash Check
print("Generating MD5 Hash ...")
new_hash = hashlib.md5(brotliDB).hexdigest()
with open(os.path.join(script_dir, "out/version.json")) as f:
old_version = json.load(f)
if old_version.get("hash") == new_hash:
print("Database Hash are same, Return")
return
print(
f"Old Hash: {old_version.get('hash')} ({old_version.get('TruthVersion')})"
)
print(f"New Hash: {new_hash} ({truthVersion})")
# Save
shutil.copyfile(os.path.join(script_dir, "out/redive_tw.db"),
os.path.join(script_dir, "out/prev.redive_tw.db"))
with open(os.path.join(script_dir, "out/redive_tw.db.br"), "wb") as f:
f.write(brotliDB)
with open(os.path.join(script_dir, "out/redive_tw.db"), "wb") as f:
f.write(masterDB)
with open(os.path.join(script_dir, "out/version.json"), "w") as f:
json.dump({"TruthVersion": truthVersion, "hash": new_hash}, f)
# Diff Check
print("Generating diff report ...")
os.system(
f"{os.path.join(script_dir, 'sqldiff.exe')} {os.path.join(script_dir, 'out/prev.redive_tw.db')} {os.path.join(script_dir, 'out/redive_tw.db')} > {os.path.join(script_dir, f'out/diff/{truthVersion}.sql')}"
)
print("Done\n")
return True
def setUpFlavorData(cls): assert os.path.exists(METADATA) with open(METADATA, 'rb') as f: cls.xml_metadata = f.read() cls.compressed_metadata = brotli.compress(cls.xml_metadata, mode=brotli.MODE_TEXT) cls.privData = bytes(bytearray(random.sample(range(0, 256), 20)))
def getWOFFCollectionData(pathOrFiles, MismatchGlyfLoca=False, reverseNames=False):
from defaultData import defaultTestData
tableChecksums = []
tableData = []
tableOrder = []
collectionDirectory = []
locaIndices = []
fonts = [TTFont(pathOrFile) for pathOrFile in pathOrFiles]
for i, font in enumerate(fonts):
index = i
if reverseNames:
index = len(fonts) - i - 1
# Make the name table unique
name = font["name"]
for namerecord in name.names:
nameID = namerecord.nameID
string = namerecord.toUnicode()
if nameID == 1:
namerecord.string = "%s %d" % (string, index)
elif nameID == 4:
namerecord.string = string.replace("Regular", "%d Regular" % index)
elif nameID == 6:
namerecord.string = string.replace("-", "%d-" % index)
tags = [i for i in sorted(font.keys()) if len(i) == 4]
if "glyf" in tags:
glyf = tags.index("glyf")
loca = tags.index("loca")
tags.insert(glyf + 1, tags.pop(loca))
tableIndices = OrderedDict()
for tag in tags:
data = transformTable(font, tag)
if MismatchGlyfLoca and tag in ("glyf", "loca"):
tableData.append([tag, data])
tableChecksums.append([tag, font.reader.tables[tag].checkSum])
tableOrder.append(tag)
tableIndex = len(tableData) - 1
tableIndices[tag] = tableIndex
if tag == "loca":
locaIndices.append(tableIndex)
else:
if [tag, data] not in tableData:
tableData.append([tag, data])
tableChecksums.append([tag, font.reader.tables[tag].checkSum])
tableOrder.append(tag)
tableIndices[tag] = tableData.index([tag, data])
collectionDirectory.append(dict(numTables=len(tableIndices), flavor=font.sfntVersion, index=tableIndices))
font.close()
del font
if MismatchGlyfLoca:
locaIndices.reverse()
for i, entry in enumerate(collectionDirectory):
entry["index"]["loca"] = locaIndices[i]
totalData = "".join([data[1][1] for data in tableData])
compData = brotli.compress(totalData, brotli.MODE_FONT)
if len(compData) >= len(totalData):
compData = totalData
directory = [dict(tag=tag, origLength=0, transformLength=0, transformFlag=0) for tag in tableOrder]
header, directory, collectionHeader, collectionDirectory, tableData = defaultTestData(directory=directory,
tableData=tableData, compressedData=compData, collectionDirectory=collectionDirectory)
data = packTestHeader(header)
data += packTestDirectory(directory, isCollection=True)
data += packTestCollectionHeader(collectionHeader)
data += packTestCollectionDirectory(collectionDirectory)
data += tableData
data = padData(data)
return data
