def main(argv):
if len(argv) < 3:
print ("Usage: %s module_json output_file no_minify" % argv[0])
return 1
module_json_file_name = argv[1]
output_file_name = argv[2]
no_minify = len(argv) > 3 and argv[3]
module_dir = path.dirname(module_json_file_name)
output = StringIO()
descriptor = None
try:
descriptor = json.loads(read_file(module_json_file_name))
except:
print "ERROR: Failed to load JSON from " + module_json_file_name
raise
# pylint: disable=E1103
scripts = descriptor.get("scripts")
assert scripts
output_root_dir = path.join(path.dirname(output_file_name), "..")
concatenate_scripts(scripts, module_dir, output_root_dir, output)
output_script = output.getvalue()
output.close()
write_file(output_file_name, output_script if no_minify else rjsmin.jsmin(output_script))
def build(self):
logging.info("Started build")
artifact_root = self.root / self.config["artifacts_dir"]
shutil.rmtree(artifact_root, ignore_errors=True)
artifact_root.mkdir()
blob_root = artifact_root / "blob"
blob_root.mkdir()
shutil.copytree(self.root / "static", artifact_root / "static")
self.tree.build_hierarchy(artifact_root, self.root / "templates")
self.tree.build_blob_hierarchy(blob_root)
jobs = self.tree.get_jobs(blob_root)
self.convert_all(jobs, self.config["pandoc_args"])
self.tree.generate_peek(blob_root, self.config["peek_length"])
with (blob_root / "index.json").open("w", encoding=ENC) as f:
json.dump(self.tree.index(), f, sort_keys=True)
f.write("\n")
for f in (artifact_root / "static").iterdir():
if f.suffix == ".js":
with f.open(encoding=ENC) as fin:
cont = fin.read()
with f.open("w", encoding=ENC) as fout:
fout.write(jsmin(cont))
fout.write("\n")
elif f.suffix == ".css":
with f.open(encoding=ENC) as fin:
cont = fin.read()
with f.open("w", encoding=ENC) as fout:
fout.write(cssmin(cont))
fout.write("\n")
with (artifact_root / "rss.xml").open("w", encoding=ENC) as f:
f.write(self.rss())
def init_build():
global config, path
if os.path.exists(path[1] + u'build'):
shutil.rmtree(path[1] + u'build')
shutil.copytree(path[1] + u'assets', path[1] + u'build')
with open(config.get(u'core', u'build'), u'w') as build:
build.write(get_html())
os.remove(path[1] + u'build/index.jinja')
for top, dirs, files in os.walk(u'build/'):
for nm in files:
if os.path.join(top, nm).split(u'.')[-1] == u"js":
with open(os.path.join(top, nm)) as f:
file_str = f.read()
file_str = jsmin(file_str)
with open(os.path.join(top, nm), u"w") as f:
f.write(file_str)
print prf(u'OK'), u"Compressing file", nm, u"..."
if os.path.join(top, nm).split(u'.')[-1] == u"css":
with open(os.path.join(top, nm)) as f:
file_str = f.read()
file_str = cssmin(file_str)
with open(os.path.join(top, nm), u"w") as f:
f.write(file_str)
print prf(u'OK'), u"Compressing file", nm, u"..."
print prf(u'OK'), u"Saved in", config.get(u'core',
u'build'), u"->", config.get(
u'head', u'title')
def generate_js(locale=None):
raw_choices, named_choices = generate_choices(locale)
js_var_name = getattr(settings, "JS_CHOICES_JS_VAR_NAME",
default_settings.JS_VAR_NAME)
js_global_object_name = getattr(settings,
"JS_CHOICES_JS_GLOBAL_OBJECT_NAME",
default_settings.JS_GLOBAL_OBJECT_NAME)
minify = getattr(settings, "JS_CHOICES_JS_MINIFY",
default_settings.JS_MINIFY)
js_content = loader.render_to_string(
"django_js_choices/choices_js.tpl",
{
"raw_choices_list": [json.dumps(x) for x in raw_choices],
"named_choices": json.dumps(named_choices),
"js_var_name": js_var_name,
"js_global_object_name": js_global_object_name,
},
)
if minify:
try:
from rjsmin import jsmin
js_content = jsmin(js_content)
except ImportError:
pass
return js_content
def init_build():
global config, path
if os.path.exists(path[1] + 'build'):
shutil.rmtree(path[1] + 'build')
shutil.copytree(path[1] + 'assets', path[1] + 'build')
with open(config.get('core', 'build'), 'w') as build:
build.write(get_html())
os.remove(path[1] + 'build/index.jinja')
for top, dirs, files in os.walk('build/'):
for nm in files:
if os.path.join(top, nm).split('.')[-1] == "js":
with open(os.path.join(top, nm)) as f:
file_str = f.read()
file_str = jsmin(file_str)
with open(os.path.join(top, nm), "w") as f:
f.write(file_str)
print(prf('OK'), "Compressing file", nm, "..." )
if os.path.join(top, nm).split('.')[-1] == "css":
with open(os.path.join(top, nm)) as f:
file_str = f.read()
file_str = cssmin(file_str)
with open(os.path.join(top, nm), "w") as f:
f.write(file_str)
print(prf('OK'), "Compressing file", nm, "..." )
print(prf('OK'), "Saved in", config.get('core', 'build'), "->", config.get('head', 'title'))
def copy_dir(self, source, dest, level=0):
'''Copies the static files from one directory to another. If this command is run, we assume the user wants to overwrite any existing files.'''
# ensure the destination exists
if not os.path.exists(dest):
os.mkdir(dest)
# go through the files in this directory
for fname in os.listdir(source):
source_path = os.path.join(source, fname)
dest_path = os.path.join(dest, fname)
ext = os.path.splitext(fname)[1].lower()
### EXPLICIT IGNORE ###
if self.ignore_file(fname):
pass
### DIRECTORIES ###
# ignore these directories
elif os.path.isdir(source_path) and fname in ( 'templates', 'views', get_setting('TEMPLATES_CACHE_DIR'), '__pycache__' ):
pass
# if a directory, create it in the destination and recurse
elif os.path.isdir(source_path):
if not os.path.exists(dest_path):
os.mkdir(dest_path)
elif not os.path.isdir(dest_path): # could be a file or link
os.unlink(dest_path)
os.mkdir(dest_path)
self.copy_dir(source_path, dest_path, level+1)
### FILES ###
# we don't do any regular files at the top level
elif level == 0:
pass
# ignore these files
elif fname in ( '__init__.py', ):
pass
# ignore these extensions
elif ext in ( '.cssm', '.jsm' ):
pass
# if a regular Javscript file, minify it
elif ext == '.js' and get_setting('MINIFY_JS_CSS', False) and JSMIN:
fin = open(source_path)
fout = open(dest_path, 'w')
fout.write(jsmin(fin.read()))
fout.close()
fin.close()
elif ext == '.css' and get_setting('MINIFY_JS_CSS', False) and CSSMIN:
fin = open(source_path)
fout = open(dest_path, 'w')
fout.write(cssmin(fin.read()))
fout.close()
fin.close()
# if we get here, it's a binary file like an image, movie, pdf, etc.
else:
shutil.copy2(source_path, dest_path)
def minified_js (code) :
"""Return minified javascript `code`.
If neither `jsmin`_ nor `rjsmin`_ is installed, `code` is returned
unchanged.
.. _`jsmin`: https://bitbucket.org/dcs/jsmin/
.. _`rjsmin`: http://opensource.perlig.de/rjsmin/
"""
jsmin = None
result = code
try :
from rjsmin import jsmin
except ImportError :
try :
### https://packages.debian.org/sid/python/python-jsmin
from jsmin import jsmin
except ImportError :
logging.warning ("Couldn't import either rjsmin nor jsmin")
if jsmin is not None :
try :
result = jsmin (code)
except Exception as exc :
logging.error ("Exception during minified_js\n %s" % (exc, ))
return pyk.encoded \
( _clean_minified_js (pyk.decoded (result, "utf-8", "latin-1"))
, "utf-8"
)
def post_process_script(self,
script,
template,
options,
session,
stdlib=True):
if stdlib:
stdlib_content = options.get("_STDLIB_")
trimmed_stdlib = self.trim_stdlib(stdlib_content, script)
script = trimmed_stdlib + script
# crappy hack for forkcmd
forkopt = copy.deepcopy(options)
forkopt.set("URL", "***K***")
forkopt.set("_JOBPATH_", "")
forkopt.set("_SESSIONPATH_", "")
forkcmd = options.get("_FORKCMD_")
forkcmd = core.loader.apply_options(forkcmd, forkopt)
options.set("_FORKCMD_", forkcmd.decode())
script = core.loader.apply_options(script, options)
# obfuscate the script!
import string
script = self.scramble(script)
# minify the script
from rjsmin import jsmin
script = jsmin(script.decode()).encode()
# obfuscation options
if options.get("OBFUSCATE"):
if options.get("OBFUSCATE") == "xor":
xor_key = core.loader.create_xor_key()
xor_script = core.loader.xor_data(script, xor_key)
script = core.loader.xor_js_file(xor_script.decode(),
xor_key).encode()
script = jsmin(script.decode()).encode()
script = template.replace(b"~SCRIPT~", script)
if session and session.encoder:
encoder = session.encoder
else:
encoder = "1252"
script = script.decode().encode("cp" + encoder)
return script
def generate(self, options):
script = '(function(options) {var context = {options: options};'
script += self.script_prepend
script += '\r\n'
script += jsmin(Path('payload/main.js').read_text(encoding='utf-8'))
script += '})(%s)' % (json.dumps(options))
return script
def load(self, name):
"""Load a configuration recursively
"""
filename = find_config(name)
self.depends.append(filename)
with open(filename) as f:
data = json.loads(jsmin(f.read()))
self.parse_dict(data)
def compile(cls,
what,
mimetype='text/javascript',
cwd=None,
uri_cwd=None,
debug=None):
if debug:
return what
return jsmin(what)
def minify_javascript(js_map):
print("Minifying JavaScript files:")
for source, dest in js_map.items():
with open(source, "r") as infile:
with open(dest, "w") as outfile:
outfile.write(rjsmin.jsmin(infile.read()))
print(f"{source} minified to {dest}")
def get_bundled(elems, attr, kind, merge):
# concat all input in the block
content = ''
# construct a bundle by converting tags to import statements or by merging the raw content
for el in elems:
# is inclusion or inline block?
if el.get(attr):
# check if we're loading an alternative source in production
file = el[attr]
if not isDevelopment and el.get('prod'):
file = el['prod']
# absolute path of the given asset
if not el.get('base-dir'):
asset = '%s/assets/%s' % (settings.BASE_DIR, file)
else:
asset = os.path.normpath(
os.path.join(settings.BASE_DIR, '../',
el['base-dir'].strip('/'), file.strip('/')))
# merged content is read from file and concatenated
if merge:
f = open(asset.replace('/', os.sep), 'r', encoding='utf8')
f.seek(0)
c = f.read()
# separate each content block with a line sep
content += c + '\n'
else:
# bundles are made up of appropriate import statements (to be imported and compiled by webpack)
if 'js' in kind:
content += 'import \'%s\';\n' % asset
else:
content += '@import \'%s\';\n' % asset
else:
# concat content held within tags after cleaning up all whitespace on each newline (consistent content regardless of indentation)
content += '\n'.join(
str(x).strip()
for x in (''.join([str(x) for x in el.contents]).splitlines()))
# compile any sass to css
if 'css' in kind:
import sass
content = sass.compile(string='%s \n %s' %
(get_sass_extras(), content))
# minify the content in production
if not isDevelopment:
if 'js' in kind:
import rjsmin
content = rjsmin.jsmin(content)
elif 'css' in kind:
import rcssmin
content = rcssmin.cssmin(content)
# content is compiled and minified (if in production)
return content
def main():
variable_name, input_filename, output_filename = sys.argv[1:]
with open(input_filename) as input_file:
input_text = input_file.read()
input_text = rjsmin.jsmin(input_text)
hex_values = ['0x{0:02x}'.format(ord(char)) for char in input_text]
const_declaration = 'const char %s[] = {\n%s\n};\n' % (
variable_name, ', '.join(hex_values))
with open(output_filename, 'w') as output_file:
output_file.write(const_declaration)
def load_option_library():
library = {}
dirs = get_config_dirs()
for d in dirs:
filename = fixpath(d) + '/options.json'
if os.path.exists(filename):
with open(filename) as f:
data = json.loads(jsmin(f.read()))
library.update(data)
return library
def read_jsfile(filename: str) -> str:
""" Read and return a minified JavaScript (.js) file """
# The file is read from the directory 'js' within the directory
# containing this __init__.py file
from rjsmin import jsmin # type: ignore
basepath, _ = os.path.split(os.path.realpath(__file__))
fpath = os.path.join(basepath, "js", filename)
with open(fpath, mode="r") as file:
return jsmin(file.read())
def main():
variable_name, input_filename, output_filename = sys.argv[1:]
with open(input_filename) as input_file:
input_text = input_file.read()
input_text = rjsmin.jsmin(input_text)
hex_values = ['0x{0:02x}'.format(ord(char)) for char in input_text]
const_declaration = 'const char %s[] = {\n%s\n};\n' % (
variable_name, ', '.join(hex_values))
with open(output_filename, 'w') as output_file:
output_file.write(const_declaration)
def render_to_response(self, context, **response_kwargs):
response = super(ElmLoadJsBaseView,
self).render_to_response(context, **response_kwargs)
response.render()
response.content = jsmin(response.content.decode('utf8'))
response['Content-Type'] = 'application/javascript'
return response
def scriptify(js_text):
'''Surrounds the text with <script></script> (if needed) and minifies the javascript'''
# I leave the <script> tags in all the html templates so our editors color them right
# get the text between <script> and </script>, if it exists
match = RE_STRIP_SCRIPT.search(js_text)
if match:
js_text = match.group(1)
# minify on the fly - rjsmin is really fast, and it decreases download time
if not settings.DEBUG:
js_text = rjsmin.jsmin(js_text)
# return the javascript
return '<script>%s</script>' % (js_text)
def output_js(self):
self.output.write('<script type="text/javascript">')
for file_path in self.files['js']:
if self.options['v']:
print('Inserting javascript from', file_path)
if self.options['m']:
self.output.write(jsmin(self.resolve_path(file_path)[0]))
else:
self.output.write(self.resolve_path(file_path)[0])
self.output.write('</script>')
# Clear file queue for scripts in body of html
self.files['js'] = []
def output_js(self):
self.output.write('<script type="text/javascript">')
for file_path in self.files['js']:
if self.options['v']:
print('Inserting javascript from', file_path)
if self.options['m']:
self.output.write(jsmin(self.resolve_path(file_path)[0]))
else:
self.output.write(self.resolve_path(file_path)[0])
self.output.write('</script>')
# Clear file queue for scripts in body of html
self.files['js'] = []
def minify_css_js(folder):
for file in os.listdir(folder):
if file.endswith(".css"):
with open(os.path.join(folder, file), 'r') as css_file:
raw_css = css_file.read()
with open(os.path.join(folder, file),
'w') as css_file_minified:
css_file_minified.write(compress(raw_css))
if file.endswith(".js"):
with open(os.path.join(folder, file), 'r') as js_file:
raw_js = js_file.read()
with open(os.path.join(folder, file), 'w') as js_file_minified:
js_file_minified.write(rjsmin.jsmin(raw_js))
def get_content(self, provider_run):
if self.template is not None:
content = self.template.render(request=provider_run.request,
context=provider_run.context)
if self.options['minify']:
if jsmin is not None:
content = jsmin(content)
else:
raise ImproperlyConfigured(
"Unable to minify {}.jsm because rjsmin is not available. Please `pip install rjsmin`."
.format(self.template_name))
return '<script type="text/javascript">{}</script>'.format(content)
return None
def minify(staticfiles_path):
for root, dirs, files in os.walk(staticfiles_path):
for file in files:
if not file.endswith('.js'):
continue
file = os.path.join(root, file)
with open(file, 'r+') as fh:
minified = rjsmin.jsmin(fh.read())
fh.truncate(0)
fh.write(minified)
if os.path.exists(file + '.gz'):
with gzip.open(file + '.gz', 'w') as fh:
fh.write(minified)
def get_template_js(self, request, context):
'''Retrieves the static and mako_rendered CSS'''
ret = []
for template in self.template_chain:
ti = getattr(template, DMP_ATTR_NAME)
if ti.js:
ret.append(ti.js) # the <script> was already created once in the constructor
if ti.jsm:
js_text = SCRIPT_RENDERERS[ti.app].render(request, ti.jsm, context.kwargs)
if JSMIN and get_setting('MINIFY_JS_CSS', False):
js_text = jsmin(js_text)
ret.append('<script>%s</script>' % js_text)
return '\n'.join(ret)
def post_process_script(self, script, stdlib=True):
if stdlib:
script = self.options.get("_STDLIB_") + script
# crappy hack for forkcmd
forkopt = copy.deepcopy(self.options)
forkopt.set("URL", "***K***")
forkopt.set("_JOBPATH_", "")
forkopt.set("_SESSIONPATH_", "")
forkcmd = self.options.get("_FORKCMD_")
forkcmd = self.loader.apply_options(forkcmd, forkopt)
self.options.set("_FORKCMD_", forkcmd.decode())
template = self.options.get("_TEMPLATE_")
script = self.loader.apply_options(script, self.options)
# obfuscate the script!
import string
script = script.replace(
b"Koadic",
''.join(random.choice(string.ascii_uppercase)
for _ in range(10)).encode())
'''
import uuid
jsfile = "/tmp/" + uuid.uuid4().hex
outfile = "/tmp/" + uuid.uuid4().hex
from subprocess import call
open(jsfile, "wb").write(script)
print("Wrote to: " + jsfile)
call(["uglifyjs", "-o", outfile, "--compress", "--mangle", "--mangle-props", "--toplevel", jsfile])
print("Outfile: " + outfile)
script = open(outfile, "rb").read()
script = script.replace(b".in", b"m222")
'''
# minify the script
from rjsmin import jsmin
script = jsmin(script.decode()).encode()
# obfuscation options
if self.stager.options.get("OBFUSCATE"):
if self.stager.options.get("OBFUSCATE") == "xor":
xor_key = self.loader.create_xor_key()
xor_script = self.loader.xor_data(script, xor_key)
script = self.loader.xor_js_file(xor_script.decode(),
xor_key).encode()
script = template.replace(b"~SCRIPT~", script)
return script
def main(argv):
if len(argv) < 3:
print('usage: %s input_file imports_dir output_file no_minify' %
argv[0])
return 1
input_file_name = argv[1]
imports_dir = argv[2]
output_file_name = argv[3]
no_minify = len(argv) > 4 and argv[4]
input_file = open(input_file_name, 'r')
input_script = input_file.read()
input_file.close()
def replace(match):
import_file_name = match.group(1)
full_path = os.path.join(imports_dir, import_file_name)
if not os.access(full_path, os.F_OK):
raise Exception(
'File %s referenced in %s not found on any source paths, '
'check source tree for consistency' %
(import_file_name, input_file_name))
import_file = open(full_path, 'r')
import_script = import_file.read()
import_file.close()
return import_script
output_script = re.sub(r'importScripts?\([\'"]([^\'"]+)[\'"]\)', replace,
input_script)
if re.search("importScripts?\(\"", output_script):
raise Exception(
'Unresolved "importScript" statements found in "%s". '
'Make sure you call "importScript" in module heads only.' %
(output_file_name))
if os.path.exists(output_file_name):
os.remove(output_file_name)
output_file = open(output_file_name, 'w')
if not no_minify:
output_script = rjsmin.jsmin(output_script)
output_file.write(output_script)
output_file.close()
# Touch output file directory to make sure that Xcode will copy
# modified resource files.
if sys.platform == 'darwin':
output_dir_name = os.path.dirname(output_file_name)
os.utime(output_dir_name, None)
def minify(staticfiles_path):
for root, dirs, files in os.walk(staticfiles_path):
for filename in files:
if not filename.endswith('.js'):
continue
filepath = os.path.join(root, filename)
with open(filepath, 'r+') as fh:
minified = rjsmin.jsmin(fh.read())
fh.seek(0)
fh.write(minified)
fh.truncate()
if os.path.exists(filepath + '.gz'):
with gzip.open(filepath + '.gz', 'w') as fh:
fh.write(minified.encode('utf-8'))
def rollup(input_path, output_path, filename, max_size):
target = join(input_path, filename)
rollup_process = subprocess.Popen(
[devtools_paths.node_path(),
devtools_paths.rollup_path()] +
['--format', 'iife', '-n', 'InspectorOverlay'] + ['--input', target],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, error = rollup_process.communicate()
if not out:
raise Exception("rollup failed: " + error)
min = rjsmin.jsmin(out)
check_size(filename, min, max_size)
write_file(join(output_path, filename), min)
def main(argv):
try:
input_path_flag_index = argv.index('--input_path')
input_path = argv[input_path_flag_index + 1]
output_path_flag_index = argv.index('--output_path')
output_path = argv[output_path_flag_index + 1]
devtools_modules = argv[1:input_path_flag_index]
except:
print('Usage: %s module_1 module_2 ... module_N --input_path <input_path> --output_path <output_path>' % argv[0])
raise
for file_name in devtools_modules:
file_content = read_file(join(input_path, file_name))
minified = rjsmin.jsmin(file_content)
write_file(join(output_path, relpath(file_name, 'front_end')), minified)
def get_template_js(self, request, context):
'''Retrieves the static and mako_rendered CSS'''
ret = []
for template in self.template_chain:
ti = getattr(template, DMP_ATTR_NAME)
if ti.js:
ret.append(
ti.js
) # the <script> was already created once in the constructor
if ti.jsm:
js_text = SCRIPT_RENDERERS[ti.app].render(
request, ti.jsm, context.kwargs)
if JSMIN and get_setting('MINIFY_JS_CSS', False):
js_text = jsmin(js_text)
ret.append('<script>%s</script>' % js_text)
return '\n'.join(ret)
def compress(self, path):
str = open(path, 'r').read()
js = unicode(str, errors='ignore')
if len(js):
try:
js = rjsmin.jsmin(js) if self.minify_enabled else js
gzip_buffer = StringIO.StringIO()
gzip_file = gzip.GzipFile(mode='wb', compresslevel=9, fileobj=gzip_buffer)
gzip_file.write(js)
gzip_file.close()
ujs = gzip_buffer.getvalue()
print "compress ratio = " + repr(float(len(js)) / float(len(ujs))) + ":1"
except:
pass
else:
return ujs
return None
def render_js(paths, js_namespace=None, minify=False, template_name="jsurls/router.js"):
"""Render the JavaScript library template with the given URL path lookup
dictionary.
"""
paths_encoded = json.dumps(paths, indent=4)
javascript = render_to_string(
template_name,
{
"namespace": js_namespace,
# add additional 4-space indent for easy reading default block code:
"paths": paths_encoded.replace("\n", "\n "),
},
)
if minify:
return jsmin(javascript)
return javascript
def main():
variable_name, input_filename, output_filename = sys.argv[1:]
with open(input_filename) as input_file:
input_text = input_file.read()
# Minify the scripts in release builds to reduce the generated string array size.
if 'Debug' not in output_filename:
if input_filename.endswith('.js'):
input_text = rjsmin.jsmin(input_text)
elif input_filename.endswith('.css'):
input_text = cssmin.cssmin(input_text)
hex_values = ['0x{0:02x}'.format(ord(char)) for char in input_text]
const_declaration = 'const unsigned char %s[] = {\n%s\n};\n' % (
variable_name, ', '.join(hex_values))
with open(output_filename, 'w') as output_file:
output_file.write(const_declaration)
def main(argv):
if len(argv) < 3:
print('usage: %s input_file imports_dir output_file no_minify' % argv[0])
return 1
input_file_name = argv[1]
imports_dir = argv[2]
output_file_name = argv[3]
no_minify = len(argv) > 4 and argv[4]
input_file = open(input_file_name, 'r')
input_script = input_file.read()
input_file.close()
def replace(match):
import_file_name = match.group(1)
full_path = os.path.join(imports_dir, import_file_name)
if not os.access(full_path, os.F_OK):
raise Exception('File %s referenced in %s not found on any source paths, '
'check source tree for consistency' %
(import_file_name, input_file_name))
import_file = open(full_path, 'r')
import_script = import_file.read()
import_file.close()
return import_script
output_script = re.sub(r'importScripts?\([\'"]([^\'"]+)[\'"]\)', replace, input_script)
if re.search("importScripts?\(\"", output_script):
raise Exception('Unresolved "importScript" statements found in "%s". '
'Make sure you call "importScript" in module heads only.' %
(output_file_name))
if os.path.exists(output_file_name):
os.remove(output_file_name)
output_file = open(output_file_name, 'w')
if not no_minify:
output_script = rjsmin.jsmin(output_script)
output_file.write(output_script)
output_file.close()
# Touch output file directory to make sure that Xcode will copy
# modified resource files.
if sys.platform == 'darwin':
output_dir_name = os.path.dirname(output_file_name)
os.utime(output_dir_name, None)
def selfcontain(html_string, base):
"""Make HTML self-contained
Take an HTML string and return an HTML string with external
dependencies as much as possible removed.
Parameter:
html_string : str, required
HTML text, possibly with links to JavaScript, CSS, and images
that would require HTTP calls for rendering
base : str, required
The original URL or filepath to use for resolving relative
links and filepaths
"""
tree = html.fromstring(html_string)
scripts = [script for script in tree.findall('.//script')
if 'src' in script.attrib]
for script in scripts:
src = script.attrib['src']
del(script.attrib['src'])
script.attrib['type'] = 'text/javascript'
contents = _fetch(src, base)
script.text = jsmin(contents)
links = [link for link in tree.findall('.//link')
if 'href' in link.attrib and
'stylesheet' == link.attrib.get('rel')]
for link in links:
href = link.attrib['href']
for key in link.attrib:
del(link.attrib[key])
link.tag = 'style'
contents = _fetch(href, base)
link.text = cssmin(contents)
icons = [icon for icon in tree.findall('.//link')
if 'href' in icon.attrib and
'icon' == icon.attrib.get('rel')]
for icon in icons:
href = icon.attrib['href']
icon.attrib['href'] = _image_to_b64(href, base)
imgs = [img for img in tree.findall('.//img')
if 'src' in img.attrib]
for img in imgs:
src = img.attrib['src']
img.attrib['src'] = _image_to_b64(src, base)
return html.tostring(tree)
def minify(staticfiles_path):
for root, dirs, files in os.walk(staticfiles_path):
for file in files:
if not file.endswith('.js'):
continue
file = os.path.join(root, file)
with open(file, 'r+') as fh:
minified = rjsmin.jsmin(fh.read())
fh.truncate(0)
fh.write(minified)
if os.path.exists(file + '.gz'):
with gzip.open(file + '.gz', 'w') as fh:
if PY2:
# On Python 2, calling encode without decoding first
# will raise a decode error because python will assume ascii
# when decoding the string to then encode it.
minified = minified.decode('utf-8')
fh.write(minified.encode('utf-8'))
def __init__(self, filename):
din = open(filename).read()
self.data = json.loads(jsmin(din))
try:
from jsonschema import Draft7Validator
# Validate configuration against schema
schemaPath = os.path.dirname(__file__) + '/schema.json'
schema = json.load(open(schemaPath))
v = Draft7Validator(schema)
errors = sorted(v.iter_errors(self.data), key=lambda e: e.path)
if errors != []:
for e in errors:
sys.stderr.write("%s @ %s\n" % (e.message, e.path))
sys.exit(1)
except ImportError as err:
sys.stderr.write(
"\n** WARNING! %s: Cannot validate config '%s', please run `make python-requirements` **\n\n"
% (str(err), filename))
def compile(cls,
what,
mimetype='text/coffeescript',
cwd=None,
uri_cwd=None,
debug=None):
args = ['--compile', '--stdio']
if cls.extra_args:
args.extend(cls.extra_args)
args.insert(0, cls.binary)
try:
handler = subprocess.Popen(args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=None)
except OSError as e:
msg = '{0} encountered an error when executing {1}: {2}'.format(
cls.__name__,
cls.binary,
u(e),
)
if e.errno == errno.ENOENT:
msg += ' Make sure {0} is in your PATH.'.format(cls.binary)
raise InvalidCompressorError(msg)
if isinstance(what, file):
what = what.read()
(stdout, stderr) = handler.communicate(input=utf8_encode(what))
stdout = u(stdout)
if not debug:
stdout = jsmin(stdout)
if handler.returncode == 0:
return stdout
else:
raise RuntimeError('Test this :S %s' % stderr)
def compress(self, path):
str = open(path, 'r').read()
js = unicode(str, errors='ignore')
if len(js):
try:
js = rjsmin.jsmin(js) if self.minify_enabled else js
gzip_buffer = StringIO.StringIO()
gzip_file = gzip.GzipFile(mode='wb',
compresslevel=9,
fileobj=gzip_buffer)
gzip_file.write(js)
gzip_file.close()
ujs = gzip_buffer.getvalue()
print "compress ratio = " + repr(
float(len(js)) / float(len(ujs))) + ":1"
except:
pass
else:
return ujs
return None
def strip_whitespace_and_comments(file_name, content):
result = re.match(r'.*\.([^.]+)', file_name)
if not result:
print 'The file name has no extension:', file_name
sys.exit(1)
extension = result.group(1).lower()
multi_line_comment = re.compile(r'/\*.*?\*/', re.MULTILINE | re.DOTALL)
repeating_space = re.compile(r'\s+', re.MULTILINE)
leading_space = re.compile(r'^\s+', re.MULTILINE)
trailing_space = re.compile(r'\s+$', re.MULTILINE)
empty_line = re.compile(r'\n+')
if extension == 'js':
content = rjsmin.jsmin(content)
elif extension == 'css':
content = multi_line_comment.sub('', content)
content = repeating_space.sub(' ', content)
content = leading_space.sub('', content)
content = trailing_space.sub('', content)
content = empty_line.sub('\n', content)
return content
def strip_whitespace_and_comments(file_name, content):
result = re.match(r'.*\.([^.]+)', file_name)
if not result:
print 'The file name has no extension:', file_name
sys.exit(1)
extension = result.group(1).lower()
multi_line_comment = re.compile(r'/\*.*?\*/', re.MULTILINE | re.DOTALL)
repeating_space = re.compile(r'\s+', re.MULTILINE)
leading_space = re.compile(r'^\s+', re.MULTILINE)
trailing_space = re.compile(r'\s+$', re.MULTILINE)
empty_line = re.compile(r'\n+')
if extension == 'js':
content = rjsmin.jsmin(content)
elif extension == 'css':
content = multi_line_comment.sub('', content)
content = repeating_space.sub(' ', content)
content = leading_space.sub('', content)
content = trailing_space.sub('', content)
content = empty_line.sub('\n', content)
return content
def minify_file(self, target, output):
"""
Minifies the target js file.
"""
filename = os.path.basename(target)
output_file = os.path.join(output, filename)
file_end = self.name_prefix + '.js'
if not output_file.endswith(file_end):
output_file = output_file.replace('.js', file_end)
rebuild = False
config_file = os.path.join(self.env.root_path, 'configs/jsminify.ini')
# when input file changed
if os.path.isfile(output_file):
if (os.path.getmtime(target) > os.path.getmtime(output_file)
# when config file exists and changed
or os.path.isfile(config_file)
and os.path.getmtime(config_file) >
os.path.getmtime(output_file)):
rebuild = True
else:
rebuild = True
if not rebuild:
return
result = None
with open(target, 'r') as fr:
result = rjsmin.jsmin(fr.read(),
self.keep_bang_comments.lower() == 'true')
if result == None:
return
with open(output_file, 'w') as fw:
fw.write(result)
print(colored('js',
'green'), self.source_dir + os.path.basename(target),
'\u27a1', self.output_dir + os.path.basename(output_file))
def render(self, context):
try:
# Create the wrap context.
pathArg = PathArgument(context, self.path)
fullPath = os.path.join(settings.JSINCLUDE_STATIC_PATH, pathArg)
wrapContext = Context({
'script': fin(fullPath),
'tagArguments': ArgumentCollection(context, self.arguments)
}, autoescape=False)
except JSIError as err:
# Something went wrong, bail out and return the error.
return err.message
# Load the wrap template.
template = loader.get_template(self.wrapPath)
result = template.render(wrapContext)
# Do not minify if in debug mode.
if settings.TEMPLATE_DEBUG:
return result
return jsmin(result)
def compile(cls, what, mimetype="text/coffeescript", cwd=None, uri_cwd=None, debug=None):
args = ["coffee", "--compile", "--stdio"]
handler = subprocess.Popen(
args, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, cwd=None
)
if isinstance(what, file):
what = what.read()
(stdout, stderr) = handler.communicate(input=utf8_encode(what))
stdout = u(stdout)
if not debug:
stdout = jsmin(stdout)
if handler.returncode == 0:
return stdout
else:
raise RuntimeError("Test this :S %s" % stderr)
def main(argv):
if len(argv) < 3:
print('usage: %s order.html input_source_dir_1 input_source_dir_2 ... '
'output_file' % argv[0])
return 1
output_file_name = argv.pop()
input_order_file_name = argv[1]
with open(input_order_file_name, 'r') as order_html:
extractor = OrderedJSFilesExtractor(order_html.read())
expander = PathExpander(argv[2:])
output = StringIO()
for input_file_name in extractor.ordered_js_files:
full_path = expander.expand(input_file_name)
if (full_path is None):
raise Exception('File %s referenced in %s not found on any source paths, '
'check source tree for consistency' %
(input_file_name, input_order_file_name))
output.write('/* %s */\n\n' % input_file_name)
input_file = open(full_path, 'r')
output.write(input_file.read())
output.write('\n')
input_file.close()
if os.path.exists(output_file_name):
os.remove(output_file_name)
output_file = open(output_file_name, 'w')
output_file.write(rjsmin.jsmin(output.getvalue()))
output_file.close()
output.close()
# Touch output file directory to make sure that Xcode will copy
# modified resource files.
if sys.platform == 'darwin':
output_dir_name = os.path.dirname(output_file_name)
os.utime(output_dir_name, None)
def build(self):
if os.path.exists('build'):
shutil.rmtree('build')
if not os.path.exists('assets') \
and not os.path.exists('sections') \
and not os.path.exists(config.file):
print (prf('FAIL'), 'Structure does not exist', '!')
sys.exit(1)
shutil.copytree('assets', 'build')
with open(config.settings.get('core', 'build'), 'w') as build:
build.write(collector.html())
os.remove('build/index.jinja')
for (top, dirs, files) in os.walk('build/'):
for nm in files:
if os.path.join(top, nm).split('.')[-1] == 'js':
with open(os.path.join(top, nm)) as f:
file_str = f.read()
file_str = jsmin(file_str)
with open(os.path.join(top, nm), 'w') as f:
f.write(file_str)
print (prf('OK'), 'Compressing file', nm, '...')
if os.path.join(top, nm).split('.')[-1] == 'css':
with open(os.path.join(top, nm)) as f:
file_str = f.read()
file_str = cssmin(file_str)
with open(os.path.join(top, nm), 'w') as f:
f.write(file_str)
print (prf('OK'), 'Compressing file', nm, '...')
print (prf('OK'), 'Saved in', config.settings.get('core', 'build'), '->',
config.settings.get('head', 'title'))
def run(self, edit):
current_file = self.view.file_name()
if current_file is None:
return None
else:
file_parts = path.splitext(current_file)
if file_parts[1] == ".js":
fileType = "js"
elif file_parts[1] == ".css":
fileType = "css"
else:
fileType = None
if fileType is None:
sublime.error_message("Please focus on the .js or .css file you wish to minify.")
return None
# do stuff
region = sublime.Region(0, self.view.size())
text = self.view.substr(region)
if fileType == "js":
result = jsmin(text)
elif fileType == "css":
result = cssmin.cssmin(text)
# write stuff
min_file_suffix = ".min"
file_name = file_parts[0] + min_file_suffix + file_parts[1]
file_path = path.join(path.dirname(current_file), file_name)
with open(file_path, "w+", 0) as min_file:
min_file.write(result)
sublime.status_message("Minipie saved %s" % (file_path))
def compile(cls, what, mimetype='text/coffeescript', cwd=None, uri_cwd=None,
debug=None):
args = ['--compile', '--stdio']
if cls.extra_args:
args.extend(cls.extra_args)
args.insert(0, cls.binary)
try:
handler = subprocess.Popen(args, stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE, cwd=None)
except OSError as e:
msg = '{0} encountered an error when executing {1}: {2}'.format(
cls.__name__,
cls.binary,
u(e),
)
if e.errno == errno.ENOENT:
msg += ' Make sure {0} is in your PATH.'.format(cls.binary)
raise InvalidCompressorError(msg)
if isinstance(what, file):
what = what.read()
(stdout, stderr) = handler.communicate(input=utf8_encode(what))
stdout = u(stdout)
if not debug:
stdout = jsmin(stdout)
if handler.returncode == 0:
return stdout
else:
raise RuntimeError('Test this :S %s' % stderr)
def minify_js(javascript):
return rjsmin.jsmin(javascript)
def output(self, _in, out, **kw):
keep = self.keep_bang_comments or False
out.write(rjsmin.jsmin(_in.read(), keep_bang_comments=keep))
def output(self, _in, out, **kw):
out.write(rjsmin.jsmin(_in.read()))
def minify_js_proc(src_text):
return jsmin(src_text)
def compress_js(self, js):
from rjsmin import jsmin
return jsmin(js)
# Generate banner
with open(args.b, 'r') as fp:
template = string.Template(fp.read())
d = date.today()
vars = dict(version=args.v,
date=d.strftime("%Y-%m-%d"),
year=d.strftime("%Y"))
banner = template.substitute(vars)
# write output to file
dir = os.path.dirname(args.m)
if not os.path.exists(dir):
os.makedirs(dir);
with open(args.m,"w") as fp:
fp.write(banner)
fp.write(output.getvalue())
# write minimized output to file
dir = os.path.dirname(args.o)
if not os.path.exists(dir):
os.makedirs(dir);
if isJavaScript:
with open(args.o,"w") as fp:
fp.write(banner)
fp.write(rjsmin.jsmin(output.getvalue()))
else:
with open(args.o,"w") as fp:
fp.write(banner)
fp.write(rcssmin.cssmin(output.getvalue()))
def GetCompressedSource(self): return rjsmin.jsmin(self.GetUncompressedSource())
def compress_js(self, js):
return rjsmin.jsmin(js, keep_bang_comments=True)
def compress_js(self, js):
return jsmin(js)
