def test_registerCSSFiles__one_slimmed_and_gzipped(self):
""" test the registerCSSFiles() with slim_if_possible=True but one
of the files shouldn't be slimmed because its filename indicates
that it's already been slimmed/packed/minified.
In this test, the big challange is when two js files are combined
and one of them should be slimmed, the other one not slimmed.
"""
if css_slimmer is None:
return
class MyProduct:
pass
files = ["test-min.css", "test.css"]
files.append(tuple(files))
registerCSSFiles(
MyProduct, files, rel_path="tests", set_expiry_header=True, slim_if_possible=True, gzip_if_possible=True
)
REQUEST = self.app.REQUEST
RESPONSE = REQUEST.RESPONSE
instance = MyProduct()
for filename in files:
if isinstance(filename, tuple):
filename = ",".join(filename)
static = getattr(instance, filename)
self.assertTrue(isinstance(static, BetterImageFile))
# if you just call static.__str__() you're not calling it
# with a REQUEST that accepts gzip encoding
REQUEST.set("HTTP_ACCEPT_ENCODING", "gzip")
bin_rendered = static.index_html(REQUEST, RESPONSE)
# expect this to be slimmed
if len(filename.split(",")) > 1:
content_parts = []
for filename in filename.split(","):
content = open(right_here(filename)).read()
if filename.find("min") > -1:
content_parts.append(content)
else:
content_parts.append(css_slimmer(content))
expected_content = "\n".join(content_parts)
expected_content = expected_content.strip()
else:
if filename.find("min") > -1:
expected_content = open(right_here(filename)).read()
else:
expected_content = css_slimmer(open(right_here(filename)).read())
expected_content = expected_content.strip()
rendered = _gzip2ascii(bin_rendered)
self.assertEqual(rendered.strip(), expected_content)
def test_registering_with_slimming_and_images_in_css(self):
try:
from slimmer import js_slimmer, css_slimmer
except ImportError:
# not possible to test this
return
class MyProduct:
pass
instance = MyProduct()
# it will only fix images that have otherwise been registered
registerImage(MyProduct, right_here("image.jpg"), rel_path="tests")
registerCSSFile(
MyProduct,
"containsimages.css",
rel_path="tests",
set_expiry_header=True,
slim_if_possible=True,
replace_images_with_aliases=True,
)
static = getattr(instance, "containsimages.css")
# Not the same...
self.assertNotEqual(str(static), css_slimmer(open(right_here("containsimages.css")).read()))
# unless you remove all '.\d+.'
self.assertEqual(
re.sub("\.\d{10,11}\.", ".", str(static)), css_slimmer(open(right_here("containsimages.css")).read())
)
# because we haven't registered large.jpg it won't be aliased
self.assertTrue("large.jpg" in str(static))
self.assertTrue("image.jpg" not in str(static))
self.assertEqual(
sorted(["containsimages.css-slimmed.css-aliased.css", "containsimages.css-slimmed.css"]),
sorted(os.listdir(_get_autogenerated_dir())),
)
# if you don it again it should just overwrite the old one
registerCSSFile(
MyProduct,
"containsimages.css",
rel_path="tests",
set_expiry_header=True,
slim_if_possible=True,
replace_images_with_aliases=True,
)
self.assertEqual(
sorted(["containsimages.css-slimmed.css-aliased.css", "containsimages.css-slimmed.css"]),
sorted(os.listdir(_get_autogenerated_dir())),
)
def test_registerCSSFiles__both_slimmed_and_gzipped(self):
""" test the registerCSSFiles() with slim_if_possible=True """
if css_slimmer is None:
return
class MyProduct:
pass
# test setting a bunch of files
files = ["large.css", "test.css"]
files.append(tuple(files))
registerCSSFiles(
MyProduct, files, rel_path="tests", set_expiry_header=True, slim_if_possible=True, gzip_if_possible=True
)
instance = MyProduct()
REQUEST = self.app.REQUEST
RESPONSE = REQUEST.RESPONSE
for filename in files:
if isinstance(filename, tuple):
filename = ",".join(filename)
static = getattr(instance, filename)
self.assertTrue(isinstance(static, BetterImageFile))
# if you just call static.__str__() you're not calling it
# with a REQUEST that accepts gzip encoding
REQUEST.set("HTTP_ACCEPT_ENCODING", "gzip")
bin_rendered = static.index_html(REQUEST, RESPONSE)
rendered = _gzip2ascii(bin_rendered).strip()
# expect this to be slimmed
if len(filename.split(",")) > 1:
content_parts = [open(right_here(x)).read() for x in filename.split(",")]
if css_slimmer is None:
expected_content = "\n".join(content_parts)
else:
expected_content = "\n".join([css_slimmer(x) for x in content_parts])
else:
content = open(right_here(filename)).read()
if css_slimmer is None:
expected_content = content
else:
expected_content = css_slimmer(content)
self.assertEqual(rendered.strip(), expected_content.strip())
def test_registerCSSFiles__one_slimmed(self):
""" test the registerCSSFiles() with slim_if_possible=True but one
of the files shouldn't be slimmed because its filename indicates
that it's already been slimmed/packed/minified.
In this test, the big challange is when two js files are combined
and one of them should be slimmed, the other one not slimmed.
"""
if css_slimmer is None:
return
class MyProduct:
pass
files = ["test-min.css", "test.css"]
files.append(tuple(files))
registerCSSFiles(
MyProduct, files, rel_path="tests", set_expiry_header=True, slim_if_possible=True, gzip_if_possible=False
)
instance = MyProduct()
for filename in files:
if isinstance(filename, tuple):
filename = ",".join(filename)
static = getattr(instance, filename)
self.assertTrue(isinstance(static, BetterImageFile))
rendered = str(static).strip()
# expect this to be slimmed partially
if len(filename.split(",")) > 1:
content_parts = []
for filename in filename.split(","):
content = open(right_here(filename)).read()
if filename.find("min") > -1:
content_parts.append(content)
else:
content_parts.append(css_slimmer(content))
expected_content = "\n".join(content_parts)
expected_content = expected_content.strip()
else:
if filename.find("min") > -1:
expected_content = open(right_here(filename)).read()
else:
expected_content = css_slimmer(open(right_here(filename)).read())
expected_content = expected_content.strip()
self.assertEqual(rendered.strip(), expected_content)
def therapy(self):
css = []
for fn in self.absolute_paths:
with open(fn, 'r') as fp:
css.append(fp.read())
css = css_slimmer(''.join(css))
storage.save(self.node.destination, ContentFile(css))
def therapy(self):
css = []
for fn in self.absolute_paths:
with open(fn, 'r') as fp:
css.append(fp.read())
css = css_slimmer(''.join(css))
storage.save(self.node.destination, ContentFile(css))
def get_code(self):
import slimmer
f = open(self.path)
code = f.read()
f.close()
if self.filetype == '.css':
return slimmer.css_slimmer(code)
if self.filetype == '.js':
return slimmer.js_slimmer(code)
if self.filetype == '.html':
return slimmer.xhtml_slimmer(code)
return code
def get_code(self):
import slimmer
f = codecs.open(self.path, 'r', 'utf-8')
code = f.read()
f.close()
if self.filetype == '.css':
return slimmer.css_slimmer(code)
if self.filetype == '.js':
return slimmer.js_slimmer(code)
if self.filetype == '.html':
return slimmer.xhtml_slimmer(code)
return code
def registerCSS(filename, path='css', slim_if_possible=True):
product = OFS.misc_.misc_.IssueTrackerMassContainer
objectid = filename
setattr(product, objectid,
BetterImageFile(os.path.join(path, filename), globals()))
obj = getattr(product, objectid)
if css_slimmer is not None and OPTIMIZE:
if slim_if_possible:
slimmed = css_slimmer(open(obj.path, 'rb').read())
new_path = obj.path + '-slimmed'
open(new_path, 'wb').write(slimmed)
setattr(obj, 'path', new_path)
def _registerCSS(product, filename, path='css', slim_if_possible=True):
objectid = filename
setattr(product, objectid,
BetterImageFile(os.path.join(path, filename), globals()))
obj = getattr(product, objectid)
if css_slimmer is not None and OPTIMIZE:
if slim_if_possible:
slimmed = css_slimmer(open(obj.path, 'rb').read())
new_path = obj.path + '-slimmed.css'
new_path = _get_autogenerated_file_path(new_path)
open(new_path, 'wb').write(slimmed)
setattr(obj, 'path', new_path)
def _get_code(self):
import slimmer
f = open(self.path)
code = f.read()
f.close()
if self.filetype == '.css':
return slimmer.css_slimmer(code)
if self.filetype == '.js':
return slimmer.js_slimmer(code)
if self.filetype == '.html':
return slimmer.xhtml_slimmer(code)
return code
def test_registering_with_slimming_and_gzip_and_images_in_css(self):
try:
from slimmer import js_slimmer, css_slimmer
except ImportError:
# not possible to test this
return
class MyProduct:
pass
instance = MyProduct()
registerJSFile(MyProduct, "test.js", rel_path="tests", set_expiry_header=True, slim_if_possible=True)
static = getattr(instance, "test.js")
self.assertEqual(str(static), js_slimmer(open(right_here("test.js")).read()))
registerCSSFile(
MyProduct,
"test.css",
rel_path="tests",
set_expiry_header=True,
slim_if_possible=True,
gzip_if_possible=True,
)
static = getattr(instance, "test.css")
self.assertEqual(str(static), css_slimmer(open(right_here("test.css")).read()))
# if you don it again it should just overwrite the old one
registerCSSFile(
MyProduct,
"test.css",
rel_path="tests",
set_expiry_header=True,
slim_if_possible=True,
gzip_if_possible=True,
)
static = getattr(instance, "test.css")
self.assertEqual(str(static), css_slimmer(open(right_here("test.css")).read()))
def optimize(content, type_):
if type_ == CSS:
if getattr(settings, 'DJANGO_STATIC_YUI_COMPRESSOR', None):
return _run_yui_compressor(content, type_)
return css_slimmer(content)
elif type_ == JS:
if getattr(settings, 'DJANGO_STATIC_CLOSURE_COMPILER', None):
return _run_closure_compiler(content)
if getattr(settings, 'DJANGO_STATIC_YUI_COMPRESSOR', None):
return _run_yui_compressor(content, type_)
return js_slimmer(content)
else:
raise ValueError("Invalid type %r" % type_)
def _registerCSS(product, filename, path='css', slim_if_possible=True):
objectid = filename
setattr(product,
objectid,
BetterImageFile(os.path.join(path, filename), globals())
)
obj = getattr(product, objectid)
if css_slimmer is not None and OPTIMIZE:
if slim_if_possible:
slimmed = css_slimmer(open(obj.path,'rb').read())
new_path = obj.path + '-slimmed.css'
new_path = _get_autogenerated_file_path(new_path)
open(new_path, 'wb').write(slimmed)
setattr(obj, 'path', new_path)
def test_registerCSSFiles__both_slimmed(self):
""" test the registerCSSFiles() with slim_if_possible=True """
if css_slimmer is None:
return
try:
class MyProduct:
pass
# test setting a bunch of files
files = ["large.css", "test.css"]
files.append(tuple(files))
registerCSSFiles(
MyProduct,
files,
rel_path="tests",
set_expiry_header=True,
slim_if_possible=True,
gzip_if_possible=False,
)
instance = MyProduct()
for filename in files:
if isinstance(filename, tuple):
filename = ",".join(filename)
static = getattr(instance, filename)
self.assertTrue(isinstance(static, BetterImageFile))
rendered = str(static)
# expect this to be slimmed
if len(filename.split(",")) > 1:
content_parts = [open(right_here(x)).read() for x in filename.split(",")]
expected_content = "\n".join([css_slimmer(x) for x in content_parts])
else:
expected_content = css_slimmer(open(right_here(filename)).read())
self.assertEqual(rendered.strip(), expected_content.strip())
except ImportError:
pass
def render(self, context):
code = self.nodelist.render(context)
if self.format == 'css':
return css_slimmer(code)
elif self.format in ('js', 'javascript'):
return js_slimmer(code)
elif self.format == 'html':
return html_slimmer(code)
else:
format = guessSyntax(code)
if format:
self.format = format
return self.render(context)
return code
def render(self, context):
code = self.nodelist.render(context)
if self.format == 'css':
return css_slimmer(code)
elif self.format in ('js', 'javascript'):
return js_slimmer(code)
elif self.format == 'html':
return html_slimmer(code)
else:
format = guessSyntax(code)
if format:
self.format = format
return self.render(context)
return code
def registerCSS(filename, path='css', slim_if_possible=True):
product = OFS.misc_.misc_.IssueTrackerMassContainer
objectid = filename
setattr(product,
objectid,
BetterImageFile(os.path.join(path, filename), globals())
)
obj = getattr(product, objectid)
if css_slimmer is not None and OPTIMIZE:
if slim_if_possible:
slimmed = css_slimmer(open(obj.path,'rb').read())
new_path = obj.path + '-slimmed'
open(new_path, 'wb').write(slimmed)
setattr(obj, 'path', new_path)
def optimize(content, type_):
if type_ == CSS:
if cssmin is not None:
return _run_cssmin(content)
elif getattr(settings, 'DJANGO_STATIC_YUI_COMPRESSOR', None):
return _run_yui_compressor(content, type_)
return slimmer.css_slimmer(content)
elif type_ == JS:
if getattr(settings, 'DJANGO_STATIC_CLOSURE_COMPILER', None):
return _run_closure_compiler(content)
if getattr(settings, 'DJANGO_STATIC_YUI_COMPRESSOR', None):
return _run_yui_compressor(content, type_)
if getattr(settings, 'DJANGO_STATIC_JSMIN', None):
return _run_jsmin(content)
return slimmer.js_slimmer(content)
else:
raise ValueError("Invalid type %r" % type_)
def test_registering_with_slimming_basic(self):
try:
from slimmer import js_slimmer, css_slimmer
except ImportError:
# not possible to test this
return
class MyProduct:
pass
instance = MyProduct()
registerJSFile(MyProduct, "test.js", rel_path="tests", set_expiry_header=True, slim_if_possible=True)
static = getattr(instance, "test.js")
self.assertEqual(str(static), js_slimmer(open(right_here("test.js")).read()))
# this will have created a file called 'test.js-slimmed.js' whose content
# is the same as str(static)
copy_test_js = os.path.join(_get_autogenerated_dir(), "test.js-slimmed.js")
self.assertTrue(os.path.isfile(copy_test_js))
self.assertEqual(open(copy_test_js).read(), str(static))
# and it that directory there should not be any other files
for f in os.listdir(os.path.dirname(copy_test_js)):
self.assertEqual(f, os.path.basename(copy_test_js))
registerCSSFile(MyProduct, "test.css", rel_path="tests", set_expiry_header=True, slim_if_possible=True)
static = getattr(instance, "test.css")
self.assertEqual(str(static), css_slimmer(open(right_here("test.css")).read()))
# this will have created a file called 'test.css-slimmed.css' whose content
# is the same as str(static)
copy_test_css = os.path.join(_get_autogenerated_dir(), "test.css-slimmed.css")
self.assertTrue(os.path.isfile(copy_test_css))
self.assertEqual(open(copy_test_css).read(), str(static))
# and it that directory there should not be any other files other
# than the one we made before called test.js-slimmed.js
for f in os.listdir(os.path.dirname(copy_test_css)):
self.assertTrue(f == os.path.basename(copy_test_js) or f == os.path.basename(copy_test_css))
# if you don it again it should just overwrite the old one
registerCSSFile(MyProduct, "test.css", rel_path="tests", set_expiry_header=True, slim_if_possible=True)
static = getattr(instance, "test.css")
# there should still only be two files in the autogenerated directory
self.assertEqual(len(os.listdir(_get_autogenerated_dir())), 2)
def optimize(content, type_):
if type_ == CSS:
if cssmin is not None:
return _run_cssmin(content)
elif getattr(settings, "DJANGO_STATIC_YUI_COMPRESSOR", None):
return _run_yui_compressor(content, type_)
return slimmer.css_slimmer(content)
elif type_ == JS:
if getattr(settings, "DJANGO_STATIC_CLOSURE_COMPILER", None):
return _run_closure_compiler(content)
if getattr(settings, "DJANGO_STATIC_YUI_COMPRESSOR", None):
return _run_yui_compressor(content, type_)
if getattr(settings, "DJANGO_STATIC_JSMIN", None):
return _run_jsmin(content)
return slimmer.js_slimmer(content)
else:
raise ValueError("Invalid type %r" % type_)
def render(self, context):
code = self.nodelist.render(context)
if slimmer is None:
return code
if self.format not in ('css','js','html','xhtml'):
self.format = guessSyntax(code)
if self.format == 'css':
return css_slimmer(code)
elif self.format in ('js', 'javascript'):
return js_slimmer(code)
elif self.format == 'xhtml':
return xhtml_slimmer(code)
elif self.format == 'html':
return html_slimmer(code)
return code
def render(self, context):
code = self.nodelist.render(context)
if slimmer is None:
return code
if self.format not in ('css','js','html','xhtml'):
self.format = slimmer.guessSyntax(code)
if self.format == 'css':
return slimmer.css_slimmer(code)
elif self.format in ('js', 'javascript'):
return slimmer.js_slimmer(code)
elif self.format == 'xhtml':
return slimmer.xhtml_slimmer(code)
elif self.format == 'html':
return slimmer.html_slimmer(code)
else:
raise TemplateSyntaxError("Unrecognized format for slimming content")
return code
def render(self, context):
code = self.nodelist.render(context)
if slimmer is None:
return code
if self.format not in ('css','js','html','xhtml'):
self.format = slimmer.guessSyntax(code)
if self.format == 'css':
return slimmer.css_slimmer(code)
elif self.format in ('js', 'javascript'):
return slimmer.js_slimmer(code)
elif self.format == 'xhtml':
return slimmer.xhtml_slimmer(code)
elif self.format == 'html':
return slimmer.html_slimmer(code)
else:
raise TemplateSyntaxError("Unrecognized format for slimming content")
return code
def render(self, context):
code = self.nodelist.render(context)
if slimmer is None:
return code
if self.format not in ("css", "js", "html", "xhtml"):
self.format = slimmer.guessSyntax(code)
if self.format == "css":
return slimmer.css_slimmer(code)
elif self.format in ("js", "javascript"):
return slimmer.js_slimmer(code)
elif self.format == "xhtml":
return slimmer.xhtml_slimmer(code)
elif self.format == "html":
return slimmer.html_slimmer(code)
else:
raise TemplateSyntaxError("Unrecognized format for slimming content")
return code
def output(self, _in, out, **kwargs):
out.write(css_slimmer(_in.read()))
def test_registering_with_slimming_and_gzip(self):
try:
from slimmer import js_slimmer, css_slimmer
except ImportError:
# not possible to test this
return
class MyProduct:
pass
instance = MyProduct()
registerJSFile(
MyProduct, "test.js", rel_path="tests", set_expiry_header=True, slim_if_possible=True, gzip_if_possible=True
)
static = getattr(instance, "test.js")
# Note, using str(static) means it does not send the Accept-Encoding: gzip
# header.
self.assertEqual(str(static), js_slimmer(open(right_here("test.js")).read()))
# this will have created a file called 'test.js-slimmed.js' whose content
# is the same as str(static)
copy_test_js = os.path.join(_get_autogenerated_dir(), "test.js-slimmed.js")
self.assertTrue(os.path.isfile(copy_test_js))
self.assertEqual(open(copy_test_js).read(), str(static))
# it would also have generated another copy called
# test.js-slimmed.js.gz
copy_test_js_gz = os.path.join(_get_autogenerated_dir(), "test.js-slimmed.js.gz")
self.assertTrue(os.path.isfile(copy_test_js_gz))
self.assertEqual(gzip.open(copy_test_js_gz).read(), str(static))
registerCSSFile(
MyProduct,
"test.css",
rel_path="tests",
set_expiry_header=True,
slim_if_possible=True,
gzip_if_possible=True,
)
static = getattr(instance, "test.css")
self.assertEqual(str(static), css_slimmer(open(right_here("test.css")).read()))
# this will have created a file called 'test.css-slimmed.css' whose content
# is the same as str(static)
copy_test_css = os.path.join(_get_autogenerated_dir(), "test.css-slimmed.css")
self.assertTrue(os.path.isfile(copy_test_css))
self.assertEqual(open(copy_test_css).read(), str(static))
# it would also have generated another copy called
# test.css-slimmed.css.gz
copy_test_css_gz = os.path.join(_get_autogenerated_dir(), "test.css-slimmed.css.gz")
self.assertTrue(os.path.isfile(copy_test_css_gz))
self.assertEqual(gzip.open(copy_test_css_gz).read(), str(static))
# if you don it AGAIN it should just overwrite the old one
registerCSSFile(
MyProduct,
"test.css",
rel_path="tests",
set_expiry_header=True,
slim_if_possible=True,
gzip_if_possible=True,
)
static = getattr(instance, "test.css")
self.assertEqual(str(static), css_slimmer(open(right_here("test.css")).read()))
def processbundle(configfile, config, bundlename, bundledata):
"""The interesting bit, load and process the bundle data"""
# Make sure all items in the array are avaliable so as not to confuse upstream applications
bundleinfo = {
"css": {
"md5": False,
"size": {"raw": 0, "min": 0, "gz": 0},
"output": {"raw": False, "min": False, "gz": False},
},
"javascript": {
"md5": False,
"size": {"raw": 0, "min": 0, "gz": 0},
"output": {"raw": False, "min": False, "gz": False},
},
}
# Javascript is processed first
if bundledata["includes"]["javascript"]:
logging.info("-Processing Javascript")
rawdata = loadfiles(configfile, config, config["javascript"], bundledata["includes"]["javascript"])
# Some info
md5 = hashlib.md5(rawdata).hexdigest()
bundleinfo["javascript"]["md5"] = md5
# Calculate the filename
filename = {}
if config["hashfilenames"] == True:
filename["raw"] = "{0}-{1}.{2}".format(bundlename, md5, "js")
filename["min"] = "{0}-{1}.{2}".format(bundlename, md5, "min.js")
filename["gz"] = "{0}-{1}.{2}".format(bundlename, md5, "min.js.gz")
else:
filename["raw"] = "{0}.{1}".format(bundlename, "js")
filename["min"] = "{0}.{1}".format(bundlename, "min.js")
filename["gz"] = "{0}.{1}".format(bundlename, "min.js.gz")
bundleinfo["javascript"]["output"] = filename
# Write the bundle file raw data
writedata(configfile, config, config["output"], filename["raw"], rawdata)
# Minify
mindata = jsmin.jsmin(rawdata)
writedata(configfile, config, config["output"], filename["min"], mindata)
# Gzip
gzdata = zlib.compress(mindata)
writedata(configfile, config, config["output"], filename["gz"], gzdata)
# Save the sizes
bundleinfo["javascript"]["size"] = {"raw": len(rawdata), "min": len(mindata), "gz": len(gzdata)}
# Now CSS (same as above, but a little different)
if bundledata["includes"]["css"]:
logging.info("-Processing CSS")
rawdata = loadfiles(configfile, config, config["css"], bundledata["includes"]["css"])
# Some info
md5 = hashlib.md5(rawdata).hexdigest()
bundleinfo["css"]["md5"] = md5
# Calculate the filename
filename = {}
if config["hashfilenames"] == True:
filename["raw"] = "{0}-{1}.{2}".format(bundlename, md5, "css")
filename["min"] = "{0}-{1}.{2}".format(bundlename, md5, "min.css")
filename["gz"] = "{0}-{1}.{2}".format(bundlename, md5, "min.css.gz")
else:
filename["raw"] = "{0}.{1}".format(bundlename, "css")
filename["min"] = "{0}.{1}".format(bundlename, "min.css")
filename["gz"] = "{0}.{1}".format(bundlename, "min.css.gz")
bundleinfo["css"]["output"] = filename
# Write the bundle file raw data
writedata(configfile, config, config["output"], filename["raw"], rawdata)
# Minify
mindata = slimmer.css_slimmer(rawdata)
writedata(configfile, config, config["output"], filename["min"], mindata)
# Gzip
gzdata = zlib.compress(mindata)
writedata(configfile, config, config["output"], filename["gz"], gzdata)
# Save the sizes
bundleinfo["css"]["size"] = {"raw": len(rawdata), "min": len(mindata), "gz": len(gzdata)}
return bundleinfo
def output(self, **kwargs):
return slimmer.css_slimmer(self.content)
def css_minifier(filedata, content_type, remote_path, is_active):
is_css = content_type == 'text/css' or remote_path.lower().endswith('.css')
if SLIMMER_INSTALLED and is_active and is_css:
return slimmer.css_slimmer(filedata)
def evaluate(self,scope, locals, block=None): if not hasattr(self,'output') or not self.output: from slimmer import css_slimmer self.output = css_slimmer(self.data) return self.output
def processbundle(configfile, config, bundlename, bundledata):
"""The interesting bit, load and process the bundle data"""
#Make sure all items in the array are avaliable so as not to confuse upstream applications
bundleinfo = {
'css': {
'md5': False,
'size': {
'raw': 0,
'min': 0,
'gz': 0
},
'output': {
'raw': False,
'min': False,
'gz': False
},
'files': []
},
'javascript': {
'md5': False,
'size': {
'raw': 0,
'min': 0,
'gz': 0
},
'output': {
'raw': False,
'min': False,
'gz': False
},
'files': []
}
}
#Javascript is processed first
if bundledata['includes']['javascript']:
logging.info('-Processing Javascript')
rawdata = loadfiles(configfile, config, config['javascript'],
bundledata['includes']['javascript'])
rawdata = '\n;'.join(rawdata)
#Some info
md5 = hashlib.md5(rawdata).hexdigest()
bundleinfo['javascript']['md5'] = md5
bundleinfo['javascript']['files'] = bundledata['includes'][
'javascript']
#Calculate the filename
filename = {}
if config['hashfilenames'] == True:
filename['raw'] = '{0}-{1}.{2}'.format(bundlename, md5, 'js')
filename['min'] = '{0}-{1}.{2}'.format(bundlename, md5, 'min.js')
filename['gz'] = '{0}-{1}.{2}'.format(bundlename, md5, 'min.js.gz')
else:
filename['raw'] = '{0}.{1}'.format(bundlename, 'js')
filename['min'] = '{0}.{1}'.format(bundlename, 'min.js')
filename['gz'] = '{0}.{1}'.format(bundlename, 'min.js.gz')
bundleinfo['javascript']['output'] = filename
#Write the bundle file raw data
writedata(configfile, config, config['output'], filename['raw'],
rawdata)
#Minify
mindata = jsmin.jsmin(rawdata)
writedata(configfile, config, config['output'], filename['min'],
mindata)
#Gzip
gzdata = zlib.compress(mindata)
writedata(configfile, config, config['output'], filename['gz'], gzdata)
#Save the sizes
bundleinfo['javascript']['size'] = {
'raw': len(rawdata),
'min': len(mindata),
'gz': len(gzdata)
}
#Now CSS (same as above, but a little different)
if bundledata['includes']['css']:
logging.info('-Processing CSS')
rawdata = loadfiles(configfile, config, config['css'],
bundledata['includes']['css'])
rawdata = ''.join(rawdata)
#Some info
md5 = hashlib.md5(rawdata).hexdigest()
bundleinfo['css']['md5'] = md5
bundleinfo['css']['files'] = bundledata['includes']['css']
#Calculate the filename
filename = {}
if config['hashfilenames'] == True:
filename['raw'] = '{0}-{1}.{2}'.format(bundlename, md5, 'css')
filename['min'] = '{0}-{1}.{2}'.format(bundlename, md5, 'min.css')
filename['gz'] = '{0}-{1}.{2}'.format(bundlename, md5,
'min.css.gz')
else:
filename['raw'] = '{0}.{1}'.format(bundlename, 'css')
filename['min'] = '{0}.{1}'.format(bundlename, 'min.css')
filename['gz'] = '{0}.{1}'.format(bundlename, 'min.css.gz')
bundleinfo['css']['output'] = filename
#Write the bundle file raw data
writedata(configfile, config, config['output'], filename['raw'],
rawdata)
#Minify
mindata = slimmer.css_slimmer(rawdata)
writedata(configfile, config, config['output'], filename['min'],
mindata)
#Gzip
gzdata = zlib.compress(mindata)
writedata(configfile, config, config['output'], filename['gz'], gzdata)
#Save the sizes
bundleinfo['css']['size'] = {
'raw': len(rawdata),
'min': len(mindata),
'gz': len(gzdata)
}
return bundleinfo
def css_minifier(filedata, content_type, remote_path, is_active):
is_css = content_type == 'text/css' or remote_path.lower().endswith('.css')
if SLIMMER_INSTALLED and is_active and is_css:
return slimmer.css_slimmer(filedata)
def js_minifier(filedata, content_type, remote_path, is_processors_active):
is_js = content_type == 'text/javascript' or remote_path.lower().endswith('.js')
if SLIMMER_INSTALLED and is_processors_active and is_js:
return slimmer.css_slimmer(filedata)
def processbundle(configfile, config, bundlename, bundledata):
"""The interesting bit, load and process the bundle data"""
#Make sure all items in the array are avaliable so as not to confuse upstream applications
bundleinfo = {
'css':{
'md5':False,
'size':{
'raw':0,
'min':0,
'gz':0
},
'output':{
'raw':False,
'min':False,
'gz':False
},
'files':[]
},
'javascript':{
'md5':False,
'size':{
'raw':0,
'min':0,
'gz':0
},
'output':{
'raw':False,
'min':False,
'gz':False
},
'files':[]
}
}
#Javascript is processed first
if bundledata['includes']['javascript']:
logging.info('-Processing Javascript')
rawdata = loadfiles(configfile, config, config['javascript'], bundledata['includes']['javascript'])
rawdata = '\n;'.join(rawdata)
#Some info
md5 = hashlib.md5(rawdata).hexdigest()
bundleinfo['javascript']['md5'] = md5
bundleinfo['javascript']['files'] = bundledata['includes']['javascript']
#Calculate the filename
filename = {}
if config['hashfilenames'] == True:
filename['raw'] = '{0}-{1}.{2}'.format(bundlename, md5, 'js')
filename['min'] = '{0}-{1}.{2}'.format(bundlename, md5, 'min.js')
filename['gz'] = '{0}-{1}.{2}'.format(bundlename, md5, 'min.js.gz')
else:
filename['raw'] = '{0}.{1}'.format(bundlename, 'js')
filename['min'] = '{0}.{1}'.format(bundlename, 'min.js')
filename['gz'] = '{0}.{1}'.format(bundlename, 'min.js.gz')
bundleinfo['javascript']['output'] = filename
#Write the bundle file raw data
writedata(configfile, config, config['output'], filename['raw'], rawdata)
#Minify
mindata = jsmin.jsmin(rawdata)
writedata(configfile, config, config['output'], filename['min'], mindata)
#Gzip
gzdata = zlib.compress(mindata)
writedata(configfile, config, config['output'], filename['gz'], gzdata)
#Save the sizes
bundleinfo['javascript']['size'] = {
'raw':len(rawdata),
'min':len(mindata),
'gz':len(gzdata)
}
#Now CSS (same as above, but a little different)
if bundledata['includes']['css']:
logging.info('-Processing CSS')
rawdata = loadfiles(configfile, config, config['css'], bundledata['includes']['css'])
rawdata = ''.join(rawdata)
#Some info
md5 = hashlib.md5(rawdata).hexdigest()
bundleinfo['css']['md5'] = md5
bundleinfo['css']['files'] = bundledata['includes']['css']
#Calculate the filename
filename = {}
if config['hashfilenames'] == True:
filename['raw'] = '{0}-{1}.{2}'.format(bundlename, md5, 'css')
filename['min'] = '{0}-{1}.{2}'.format(bundlename, md5, 'min.css')
filename['gz'] = '{0}-{1}.{2}'.format(bundlename, md5, 'min.css.gz')
else:
filename['raw'] = '{0}.{1}'.format(bundlename, 'css')
filename['min'] = '{0}.{1}'.format(bundlename, 'min.css')
filename['gz'] = '{0}.{1}'.format(bundlename, 'min.css.gz')
bundleinfo['css']['output'] = filename
#Write the bundle file raw data
writedata(configfile, config, config['output'], filename['raw'], rawdata)
#Minify
mindata = slimmer.css_slimmer(rawdata)
writedata(configfile, config, config['output'], filename['min'], mindata)
#Gzip
gzdata = zlib.compress(mindata)
writedata(configfile, config, config['output'], filename['gz'], gzdata)
#Save the sizes
bundleinfo['css']['size'] = {
'raw':len(rawdata),
'min':len(mindata),
'gz':len(gzdata)
}
return bundleinfo
def getMinCss(cssFiles):
allCss = getAllCssAsString(cssFiles)
from slimmer import css_slimmer
minCss = css_slimmer(allCss)
return minCss
mimetype = "text/html"
else:
mimetype = "application/octet-stream"
# get raw file data
with open(filepath, "rb") as fr:
filedata = fr.read()
oldfilesize = len(filedata)
# can I remove CR, LF, Tabs?
if do_slimmer:
if fileext in [".tpl", ".html", ".htm"]:
filedata = slimmer.html_slimmer(filedata)
elif fileext in [".css"]:
filedata = slimmer.css_slimmer(filedata)
elif fileext in [".js"]:
filedata = slimmer.js_slimmer(filedata)
print "Adding {} mimetype = ({}) size = {} reduced size = {}".format(filename, mimetype, oldfilesize, len(filedata))
# filename length, mime tpye length, file content length
fw.write(struct.pack("<BBH", len(filename) + 1, len(mimetype) + 1, len(filedata)))
# filename data
fw.write(struct.pack(str(len(filename)) + "sB", filename, 0x00))
# mime type data
fw.write(struct.pack(str(len(mimetype)) + "sB", mimetype, 0x00))
# file data
def evaluate(self, scope, locals, block=None):
if not hasattr(self, 'output') or not self.output:
from slimmer import css_slimmer
self.output = css_slimmer(self.data)
return self.output
def compress(self, css):
lint = self.lintian(css)
if lint.validate(ignore_hacks=True):
return css_slimmer(css)
return css
def output(self, _in, out, **kwargs):
out.write(css_slimmer(_in.read()))
def output(self, **kwargs):
return slimmer.css_slimmer(self.content)
reverse=True,
key=lambda pair: pair[0]))
write_index(files.values())
in_feed = sorted(files.values(), reverse=True,
key=lambda item: item.date)[0:10]
for item in in_feed:
fe = fg.add_entry()
full_url = feed_root + url_of(item)
fe.id(full_url)
fe.link(href=full_url, rel='alternate')
fe.title(item.title)
fe.content(markdown(item.content))
fe.updated(tz.localize(datetime.fromtimestamp(item.mtime)))
fe.published(tz.localize(datetime.fromtimestamp(item.mtime)))
os.makedirs('out/feed/atom')
os.makedirs('out/feed/rss')
fg.atom_file('out/feed/atom/index.xml')
fg.rss_file('out/feed/rss/index.xml')
fg.rss_file('out/feed/index.xml')
for static in ['main.css']:
with open_out('static/' + static) as f:
f.write(slimmer.css_slimmer(templates.get_template(static).render()))
shutil.copytree('images', 'out/images')
def filter_css(self, css):
return slimmer.css_slimmer(css)
mimetype = "text/html"
else:
mimetype = "application/octet-stream"
# get raw file data
with open(filepath, "rb") as fr:
filedata = fr.read()
oldfilesize = len(filedata)
# can I remove CR, LF, Tabs?
if do_slimmer:
if fileext in [".tpl", ".html", ".htm"]:
filedata = slimmer.html_slimmer(filedata)
elif fileext in [".css"]:
filedata = slimmer.css_slimmer(filedata)
elif fileext in [".js"]:
filedata = slimmer.js_slimmer(filedata)
print "Adding {} mimetype = ({}) size = {} reduced size = {}".format(
filename, mimetype, oldfilesize, len(filedata))
# flags
fw.write(struct.pack("B", 0))
# filename length, mime tpye length, file content length
fw.write(
struct.pack("<BBH",
len(filename) + 1,
len(mimetype) + 1, len(filedata)))
import slimmer
import sys
if __name__ == '__main__':
for fpath in sys.argv[1:]:
with open(fpath, 'r') as fh:
txt = fh.read()
if fpath.endswith('.html'):
txt = slimmer.html_slimmer(txt)
elif fpath.endswith('.css'):
txt = slimmer.css_slimmer(txt)
elif fpath.endswith('.js'):
txt = slimmer.js_slimmer(txt)
else:
print('unknown format of', fpath)
with open(fpath, 'w') as fh:
fh.write(txt)
def slimmer(self, hard=False):
content = self.readFiles()
return css_slimmer(content, hard)
def registerCSSFile(product, filename, epath=None, Globals=globals(),
rel_path='css',
slim_if_possible=True, gzip_if_possible=False,
set_expiry_header=False,
max_age_development=60, max_age_production=3600,
expand_data64=False,
replace_images_with_aliases=False):
p_home = package_home(Globals) # product home
if filename.count('-slimmed.css'):
raise SystemError, "Again!??!"
objectid = filename
path = "%s/" % rel_path
if epath:
path = "%s/%s/" % (rel_path, epath)
filepath = '%s%s' % (path, filename)
if len(filename.split(',')) > 1:
# it's a combo name!!
real_filepath = _getAutogeneratedFilepath(os.path.join(p_home, filepath))
out = open(real_filepath, 'w')
mtimes = []
for filepath_ in [os.path.join(p_home, '%s%s' % (path, x))
for x in filename.split(',')]:
content = open(filepath_).read()
if slim_if_possible and css_slimmer and not dont_slim_file(filepath_):
content = css_slimmer(content)
out.write(content+'\n')
mtimes.append(os.stat(filepath_)[stat.ST_MTIME])
out.close()
filepath = real_filepath
mtime = max(mtimes)
# since we've taken the slimming thought into account already
# here in the loop there is no need to consider slimming the
# content further down.
slim_if_possible = False
else:
mtime = os.stat(os.path.join(p_home, filepath))[stat.ST_MTIME]
setattr(product,
objectid,
BetterImageFile(filepath, Globals,
set_expiry_header=set_expiry_header,
max_age_development=max_age_development,
max_age_production=max_age_production)
)
obj = getattr(product, objectid)
if slim_if_possible and dont_slim_file(os.path.join(p_home, filepath)):
slim_if_possible = False
if slim_if_possible and \
os.path.isfile(os.path.join(p_home, filepath)):
if os.path.isfile(os.path.join(p_home, filepath+'.nogzip')):
gzip_if_possible = False
if css_slimmer is not None and slim_if_possible:
slimmed = css_slimmer(open(obj.path,'rb').read())
filepath = _getAutogeneratedFilepath(obj.path + '-slimmed.css')
open(filepath, 'wb').write(slimmed)
setattr(obj, 'path', filepath)
if expand_data64:
# If this is true, then try to convert things like
# 'DATA64(www/img.gif)' into
# 'data:image/gif;base64,ASIDJADIJAIW00100.... ...\n'
# This feature is highly experimental and has proved problematic
# since it sometimes just doesn't work and it's really hard to
# debug why certain images aren't working. Use carefully.
#
# The spec is here: http://tools.ietf.org/html/rfc2397
# The inspiration came from here:
# http://developer.yahoo.com/performance/rules.html#num_http
new_content = False
content = file(obj.path).read()
for whole_tag, path, extension in DATA64_REGEX.findall(content):
fp = os.path.join(p_home, path)
content = content.replace(whole_tag,
'data:image/%s;base64,%s' % \
(extension,
encodestring(file(fp,'rb').read()).replace('\n','\\n')))
new_content = content
if new_content:
filepath = _getAutogeneratedFilepath(obj.path + '-data64expanded')
open(filepath, 'wb').write(new_content)
setattr(obj, 'path', filepath)
if replace_images_with_aliases:
# This feature opens the CSS content and looks for things like
# url(/misc_/MyProduct/close.png) and replaces that with
# url(/misc_/MyProduct/close.1223555.png) if possible.
new_content = False
if getattr(product, 'misc_infinite_aliases', {}):
content = file(obj.path).read()
images = []
if content.count('/misc_/'):
filenames = []
for org, alias in product.misc_infinite_aliases.items():
if anyTrue(org.endswith, ('.gif','.jpg','.png')):
filenames.append(org)
regex = 'url\(["\']*/misc_/%s/(' % product + \
'|'.join([re.escape(x) for x in filenames]) + \
')["\']*\)'
regex = re.compile(regex)
def replacer(g):
whole = g.group()
better_filename = product.misc_infinite_aliases.get(g.groups()[0], g.groups()[0])
whole = whole.replace(g.groups()[0], better_filename)
return whole
new_content = regex.sub(replacer, content)
else:
def replacer(match):
filepath, filename = match.groups()[0].rsplit('/', 1)
try:
image_product = _find_related_context(product, filepath)
except AttributeError:
#import warnings
import logging
logging.warn("Unable to find image product of %s from %r" % \
(product, filepath))
return match.group()
aliased = getattr(image_product, 'misc_infinite_aliases',
{}).get(filename)
if aliased:
return match.group().replace(filename, aliased)
else:
return match.group()
#new_content = content
new_content = referred_css_images_regex.sub(replacer, content)
if new_content:
filepath = _getAutogeneratedFilepath(obj.path + '-aliased.css')
open(filepath, 'wb').write(new_content)
setattr(obj, 'path', filepath)
# set up an alias too with near infinit max_age
a, b = os.path.splitext(filename)
objectid_alias = a + '.%s' % mtime + b
setattr(product,
objectid_alias,
BetterImageFile(obj.path, Globals,
set_expiry_header=set_expiry_header,
max_age_development=EXPIRY_INFINITY, # 5 years
max_age_production=EXPIRY_INFINITY # 5 years
)
)
# make a note of the alias
if hasattr(product, 'misc_infinite_aliases'):
aliases = product.misc_infinite_aliases
else:
aliases = {}
if objectid in aliases:
# this is not supposed to happen in the same sense that you can't have
# two files by the same name in the same directory
if getEnvBool('PREVENT_DUPLICATE_STATIC_STORAGE', False):
# by default we don't worry about that much
raise ValueError, "%r is already defined in %s.misc_infinite_aliases" %\
(objectid, product)
aliases[objectid] = objectid_alias
setattr(product, 'misc_infinite_aliases', aliases)
if gzip_if_possible:
setattr(product, objectid,
GzippedFile(filepath, Globals,
set_expiry_header=set_expiry_header,
max_age_development=max_age_development,
max_age_production=max_age_production)
)
# also set up the alias which overwrites the previously
# set up alias.
setattr(product, objectid_alias,
GzippedFile(filepath, Globals,
set_expiry_header=set_expiry_header,
max_age_development=EXPIRY_INFINITY,
max_age_production=EXPIRY_INFINITY)
)
href = link.get('href')
content = urllib.urlopen(href).read()
all_style += content
# Remove this link, we'll add the minified contents later
html = html.replace(str(link), '')
# Replace all style tags with minified style
for style in soup.findAll('style'):
content = style.text
all_style += content
# Remove this style tag, we'll add the minified contents later
html = html.replace(str(style), '')
total_size += len(all_style.encode('utf-8'))
all_style += '</style>'
all_minified_style = css_slimmer(all_style)
html = html.replace('</head>', '%s</head>' % all_minified_style)
# Replace all script tags with minified script
all_script = '<script>'
for script in soup.findAll('script'):
src = script.get('src')
if (src != None):
all_script += urllib.urlopen(src).read()
else:
all_script += script.text
html = html.replace(str(script), '')
total_size += len(all_script.encode('utf-8'))
all_script += '</script>'
all_minified_script = jsmin(all_script)
