"""A basic pool of worker threads"""

class Worker(Thread):

def __init__(self, tasks):

Thread.__init__(self)

self.tasks = tasks

self.daemon = True # threads will be killed on exit

self.start()

def run(self):

while True:

# block until a task is ready to be done

function, args, kargs = self.tasks.get()

try:

function(*args, **kargs)

except:

print("ERROR")

for count, thing in enumerate(args):

print '{0}. {1}'.format(count, thing)

print(sys.exc_info()[0])

print("ERROR")

self.tasks.errors = True

self.tasks.task_done()

def __init__(self, num_threads = multiprocessing.cpu_count() + 1):

self.tasks = Queue(num_threads)

self.tasks.errors = False

# create num_threads Workers, by default the number of CPUs + 1

for _ in range(num_threads): self.Worker(self.tasks)

def add_task(self, function, *args, **kargs):

self.tasks.put((function, args, kargs))

def wait_completion(self):

# wait for the queue to be empty

self.tasks.join()

stringio = StringIO()

gzipfile = gzip.GzipFile(t.name, 'w', 9, stringio)

with open(t.name, 'rb') as f:

shutil.copyfileobj(f, gzipfile)

gzipfile.close()

rawsize = os.stat(t.name).st_size

gzipsize = len(stringio.getvalue())

savings = '{0:.2%}'.format((rawsize - gzipsize)/float(rawsize))

t.info('uncompressed: %8d bytes', rawsize)

@target(glsl_src.replace('.glsl', 'shader.js'), glsl_src,

'src/ol/webgl/shader.mustache', 'bin/pyglslunit.py')

def shader_src(t):

t.run('%(PYTHON)s', 'bin/pyglslunit.py',

'--input', glsl_src,

'--template', 'src/ol/webgl/shader.mustache',

requires = set()

for dependency in task.dependencies:

for line in open(dependency, 'rU'):

match = re.match(r'goog\.provide\(\'(.*)\'\);', line)

if match:

requires.add(match.group(1))

with open(task.name, 'wb') as f:

for require in sorted(requires):

def action(t):

t.run('node', 'tasks/build-examples.js')

dependencies = ['examples/%(filepath)s' % match.groupdict()]

def action(t):

# When compiling the ol3 code and the application code together it is

# better to use oli.js and olx.js files as "input" files rather than

# "externs" files. Indeed, externs prevent renaming, which is neither

# necessary nor desirable in this case.

#

# oli.js and olx.js do not provide or require namespaces (using

# "goog.provide" or "goog.require"). For that reason, if they are

# specified as input files through the "src" property, then

# closure-util will exclude them when creating the dependencies graph.

# So the compile "js" property is used instead. With that property the

# oli.js and olx.js files are passed directly to the compiler. And by

# setting "manage_closure_dependencies" to "true" the compiler will not

# exclude them from its dependencies graph.

content = json.dumps({

"exports": [],

"src": [

"src/**/*.js",

"build/ol.ext/*.js",

"build/examples/%(id)s.js" % match.groupdict()],

"compile": {

"js": [

"externs/olx.js",

"externs/oli.js",

],

"externs": [

"externs/bingmaps.js",

"externs/bootstrap.js",

"externs/closure-compiler.js",

"externs/example.js",

"externs/fastclick.js",

"externs/geojson.js",

"externs/jquery-1.9.js",

"externs/proj4js.js",

"externs/tilejson.js",

"externs/topojson.js",

"externs/vbarray.js"

],

"define": [

"goog.array.ASSUME_NATIVE_FUNCTIONS=true",

"goog.dom.ASSUME_STANDARDS_MODE=true",

"goog.json.USE_NATIVE_JSON=true",

"goog.DEBUG=false"

],

"jscomp_error": [

"accessControls",

"ambiguousFunctionDecl",

"checkDebuggerStatement",

"checkEventfulObjectDisposal",

"checkRegExp",

"checkStructDictInheritance",

"checkTypes",

"checkVars",

"const",

"constantProperty",

"deprecated",

"duplicate",

"duplicateMessage",

"es3",

"es5Strict",

"externsValidation",

"fileoverviewTags",

"globalThis",

"internetExplorerChecks",

"invalidCasts",

"misplacedTypeAnnotation",

"missingProperties",

"missingProvide",

"missingRequire",

"missingReturn",

"newCheckTypes",

"nonStandardJsDocs",

"strictModuleDepCheck",

"suspiciousCode",

"typeInvalidation",

"tweakValidation",

"undefinedNames",

"undefinedVars",

"uselessCode",

"violatedModuleDep",

"visibility"

],

"jscomp_off": [

"unknownDefines"

],

"extra_annotation_name": [

"api", "observable"

],

"compilation_level": "ADVANCED",

"warning_level": "VERBOSE",

"output_wrapper": "(function(){%output%})();",

"use_types_for_optimization": True,

"manage_closure_dependencies": True

}

})

with open(t.name, 'wb') as f:

f.write(content)

return Target(name, action=action,

def action(t):

config = 'build/compiled-examples/%(id)s.json' % match.groupdict()

t.run('node', 'tasks/build.js', config, name)

report_sizes(t)

dependencies = [SRC, SHADER_SRC,

'examples/%(id)s.js' % match.groupdict(),

'build/compiled-examples/%(id)s.json' % match.groupdict(),

NPM_INSTALL]

t.run('%(GJSLINT)s',

'--jslint_error=all',

'--custom_jsdoc_tags=event,fires,function,classdesc,api,observable',

'--strict',

t.newer(t.dependencies))

# FIXME this is a horribe hack, we should use a proper JavaScript parser

# here

in_multiline_comment = False

lineno = 0

for line in lines:

lineno += 1

if in_multiline_comment:

index = line.find('*/')

if index != -1:

in_multiline_comment = False

line = line[index + 2:]

if not in_multiline_comment:

line = re.sub(r'//[^\n]*', '', line)

line = re.sub(r'/\*.*?\*/', '', line)

index = line.find('/*')

if index != -1:

yield lineno, line[:index]

in_multiline_comment = True

else:

unused_count = 0

all_provides = set()

closure_lib_path = output('node', '-e',

'process.stdout.write(require("closure-util").getLibraryPath())')

for filename in ifind(closure_lib_path):

if filename.endswith('.js'):

if not re.match(r'.*/closure/goog/', filename):

continue

# Skip goog.i18n because it contains so many modules that it causes

# the generated regular expression to exceed Python's limits

if re.match(r'.*/closure/goog/i18n/', filename):

continue

for line in open(filename, 'rU'):

m = re.match(r'goog.provide\(\'(.*)\'\);', line)

if m:

all_provides.add(m.group(1))

for filename in sorted(t.dependencies):

require_linenos = {}

uses = set()

lines = open(filename, 'rU').readlines()

for lineno, line in _strip_comments(lines):

m = re.match(r'goog.provide\(\'(.*)\'\);', line)

if m:

all_provides.add(m.group(1))

continue

m = re.match(r'goog.require\(\'(.*)\'\);', line)

if m:

require_linenos[m.group(1)] = lineno

continue

ignore_linenos = require_linenos.values()

for lineno, line in enumerate(lines):

if lineno in ignore_linenos:

continue

for require in require_linenos.iterkeys():

if require in line:

uses.add(require)

for require in sorted(set(require_linenos.keys()) - uses):

t.info('%s:%d: unused goog.require: %r' % (

filename, require_linenos[require], require))

unused_count += 1

all_provides.discard('ol')

all_provides.discard('ol.MapProperty')

class Node(object):

def __init__(self):

self.present = False

self.children = {}

def _build_re(self, key):

if key == '*':

assert len(self.children) == 0

# We want to match `.doIt` but not `.SomeClass` or `.more.stuff`

return '(?=\\.[a-z]\\w*\\b(?!\\.))'

elif len(self.children) == 1:

child_key, child = next(self.children.iteritems())

child_re = child._build_re(child_key)

if child_key != '*':

child_re = '\\.' + child_re

if self.present:

return key + '(' + child_re + ')?'

else:

return key + child_re

elif self.children:

children_re = '(?:' + '|'.join(

('\\.' if k != '*' else '') + self.children[k]._build_re(k)

for k in sorted(self.children.keys())) + ')'

if self.present:

return key + children_re + '?'

else:

return key + children_re

else:

assert self.present

return key

def build_re(self, key):

return re.compile('\\b' + self._build_re(key) + '\\b')

root = Node()

for provide in all_provides:

node = root

for component in provide.split('.'):

if component not in node.children:

node.children[component] = Node()

node = node.children[component]

if component[0].islower():

# We've arrived at a namespace provide like `ol.foo`.

# In this case, we want to match uses like `ol.foo.doIt()` but

# not match things like `new ol.foo.SomeClass()`.

# For this purpose, we use the special wildcard key for the child.

node.children['*'] = Node()

else:

node.present = True

provide_res = [child.build_re(key)

for key, child in root.children.iteritems()]

missing_count = 0

for filename in sorted(t.dependencies):

provides = set()

requires = set()

uses = set()

uses_linenos = {}

for lineno, line in _strip_comments(open(filename, 'rU')):

m = re.match(r'goog.provide\(\'(.*)\'\);', line)

if m:

provides.add(m.group(1))

continue

m = re.match(r'goog.require\(\'(.*)\'\);', line)

if m:

requires.add(m.group(1))

continue

while True:

for provide_re in provide_res:

m = provide_re.search(line)

if m:

uses.add(m.group())

uses_linenos[m.group()] = lineno

line = line[:m.start()] + line[m.end():]

break

else:

break

if filename == 'src/ol/renderer/layerrenderer.js':

uses.discard('ol.renderer.Map')

m = re.match(

r'src/ol/renderer/(\w+)/\1(\w*)layerrenderer\.js\Z', filename)

if m:

uses.discard('ol.renderer.Map')

uses.discard('ol.renderer.%s.Map' % (m.group(1),))

missing_requires = uses - requires - provides

if missing_requires:

for missing_require in sorted(missing_requires):

t.info("%s:%d missing goog.require('%s')" %

(filename, uses_linenos[missing_require], missing_require))

missing_count += 1

if unused_count or missing_count:

t.error('%d unused goog.requires, %d missing goog.requires' %

(unused_count, missing_count))

CR_RE = re.compile(r'\r')

LEADING_WHITESPACE_RE = re.compile(r'\s+')

TRAILING_WHITESPACE_RE = re.compile(r'\s+\n\Z')

NO_NEWLINE_RE = re.compile(r'[^\n]\Z')

ALL_WHITESPACE_RE = re.compile(r'\s+\Z')

errors = 0

for filename in sorted(t.newer(t.dependencies)):

whitespace = False

for lineno, line in enumerate(open(filename, 'rU')):

if lineno == 0 and LEADING_WHITESPACE_RE.match(line):

t.info('%s:%d: leading whitespace', filename, lineno + 1)

errors += 1

if CR_RE.search(line):

t.info('%s:%d: carriage return character in line', filename, lineno + 1)

errors += 1

if TRAILING_WHITESPACE_RE.search(line):

t.info('%s:%d: trailing whitespace', filename, lineno + 1)

errors += 1

if NO_NEWLINE_RE.search(line):

t.info('%s:%d: no newline at end of file', filename, lineno + 1)

errors += 1

whitespace = ALL_WHITESPACE_RE.match(line)

if whitespace:

t.info('%s: trailing whitespace at end of file', filename)

errors += 1

if errors:

t.error('%d whitespace errors' % (errors,))

t.run('%(JSDOC)s', 'config/jsdoc/api/index.md',

'-c', 'config/jsdoc/api/conf.json',

'-d', 'build/hosted/%(BRANCH)s/apidoc')

lines = open(example, 'rU').readlines()

target_lines = []

target_require_lines = []

found_requires = False

found_code = False

for line in lines:

m = re.match(r'goog.require\(\'(.*)\'\);', line)

if m:

found_requires = True

target_require_lines.append(line)

elif found_requires:

if found_code or line not in ('\n', '\r\n'):

found_code = True

target_lines.append(line)

target = open(

os.path.join(dst_dir, os.path.basename(example)), 'wb')

target_require = open(

os.path.join(dst_dir, os.path.basename(example)

.replace('.js', '-require.js')),

'wb')

target.writelines(target_lines)

target.close()

target_require.writelines(target_require_lines)

examples_dir = 'build/hosted/%(BRANCH)s/examples'

build_dir = 'build/hosted/%(BRANCH)s/build'

css_dir = 'build/hosted/%(BRANCH)s/css'

closure_lib_path = output('node', '-e',

'process.stdout.write(require("closure-util").getLibraryPath())')

t.rm_rf(examples_dir)

t.cp_r('build/examples', examples_dir)

for example in EXAMPLES_SRC_JS:

split_example_file(example, examples_dir % vars(variables))

t.cp('bin/loader_hosted_examples.js', examples_dir + '/loader.js')

t.rm_rf(build_dir)

t.makedirs(build_dir)

t.rm_rf(css_dir)

t.makedirs(css_dir)

t.cp('build/ol.js', 'build/ol-debug.js', build_dir)

t.cp('build/ol.css', css_dir)

t.rm_rf('build/hosted/%(BRANCH)s/closure-library')

t.cp_r(closure_lib_path, 'build/hosted/%(BRANCH)s/closure-library')

t.rm_rf('build/hosted/%(BRANCH)s/ol')

t.makedirs('build/hosted/%(BRANCH)s/ol')

t.cp_r('src/ol', 'build/hosted/%(BRANCH)s/ol/ol')

t.rm_rf('build/hosted/%(BRANCH)s/ol.ext')

t.cp_r('build/ol.ext', 'build/hosted/%(BRANCH)s/ol.ext')

t.run('%(PYTHON)s', closure_lib_path + '/closure/bin/build/depswriter.py',

'--root_with_prefix', 'src ../../../ol',

'--root_with_prefix', 'build/ol.ext ../../../ol.ext',

'--root', 'build/hosted/%(BRANCH)s/closure-library/closure/goog',

'--root_with_prefix', 'build/hosted/%(BRANCH)s/closure-library/'

'third_party ../../third_party',

examples = ['build/hosted/%(BRANCH)s/' + e

for e in EXAMPLES_SRC_HTML

if not open(e.replace('.html', '.js'), 'rU').readline().startswith('// NOCOMPILE')]

all_examples = [e + '?mode=advanced' for e in examples]

# Run the examples checks in a pool of threads

pool = ThreadPool()

for example in all_examples:

pool.add_task(t.run, '%(PHANTOMJS)s', '--ssl-protocol=any',

'--ignore-ssl-errors=true', 'bin/check-example.js', example)

errors = pool.wait_completion()

if errors:

# create a temp. profile to run the tests with WebGL

tmp_profile_dir = 'build/slimerjs-profile'

t.rm_rf(tmp_profile_dir)

t.cp_r('test_rendering/slimerjs-profile', tmp_profile_dir)

regex = re.compile('FIXME|TODO')

matches = dict()

totalcount = 0

for filename in SRC:

f = open(filename, 'r')

for lineno, line in enumerate(f):

if regex.search(line):

if (filename not in matches):

matches[filename] = list()

matches[filename].append('#%-10d %s' % (

lineno + 1, line.strip()))

totalcount += 1

f.close()

for filename in matches:

num_matches = len(matches[filename])

noun = 'matches' if num_matches > 1 else 'match'

print ' %s has %d %s:' % (filename, num_matches, noun)

for match in matches[filename]:

print ' %s' % (match,)

print

"""Removes untracked files and folders from previous builds."""

# -X => only clean up files that are usually ignored e.g.

# through .gitignore

# -d => also consider directories for deletion

# -f => if git configuration variable clean.requireForce != false,

# git clean will refuse to run unless given -f or -n.

for exe in EXECUTABLES:

status = 'present' if which(exe) else 'MISSING'

print 'Program "%s" seems to be %s.' % (exe, status)

print '''

build.py - The OpenLayers 3 build script.

Usage:

./build.py [options] [target] (on Unix-based machines)

<python-executable.exe> build.py [options] [target] (on Windows machines)

There is one option:

-c - Cleans up the repository from previous builds.

The most common targets are:

serve - Serves files, on port 3000.

lint - Runs gjslint on all sourcefiles to enforce specific syntax.

build - Builds singlefile versions of OpenLayers JavaScript and

CSS. This is also the default build target which runs when

no target is specified.

test - Runs the testsuite and displays the results.

test-rendering - Runs the rendering testsuite and displays the results.

check - Runs the lint-target, builds some OpenLayers files, and

then runs test. Many developers call this target often

while working on the code.

help - Shows this help.

Other less frequently used targets are:

apidoc - Builds the API-Documentation using JSDoc3.

ci - Builds all examples in various modes and usually takes a

long time to finish. This target calls the following

targets: 'lint', 'build', 'test', 'test-rendering',

'build/compiled-examples/all.combined.js', 'check-examples',

and 'apidoc'. This is the target run on Travis CI.

test-coverage - Generates a test coverage report in the coverage folder.

reallyclean - Remove untracked files from the repository.

checkdeps - Checks whether all required development software is

installed on your machine.

fixme - Will print a list of parts of the code that are marked

with either TODO or FIXME.

todo - This is an alias for the fixme-target (see above).

If no target is given, the build-target will be executed.

The above list is not complete, please see the source code for not-mentioned

and only seldom called targets.