def compile(compiler_jar_path, deps, inputs, compiler_flags, root, update_build_version=True):
input = list(inputs)[0]
# Make sure a .jar is specified.
if not compiler_jar_path:
logging.error('--compiler_jar flag must be specified if --output is '
'"compiled"')
return None
compiled_source = jscompiler.Compile(
compiler_jar_path,
deps,
compiler_flags)
if compiled_source is None:
logging.error('JavaScript compilation failed.')
return None
else:
logging.info('JavaScript compilation succeeded.')
file_name_no_suffix = os.path.splitext(os.path.basename(input))[0]
min_js = os.path.dirname(input) + "/" + file_name_no_suffix + ".c.min.js"
timestamp, build_path = version.updater.get_build_version(input, update_build_version)
dir = root.replace("\\", "/") + version.updater.BASE_BUILD_DIR + build_path + "/"
min_js_new = dir + file_name_no_suffix + "." + timestamp + ".c.min.js"
if not os.path.exists(dir):
os.makedirs(dir)
svn.try_lock(min_js_new)
out = open(min_js_new, "w")
out.write(compiled_source)
logging.info('min js : ' + min_js_new)
return min_js_new, min_js
def compile(input, output=None, flags=None):
"""Prepares command-line call to Closure Compiler.
Args:
source_paths: Source paths to build, in order.
Returns:
The compiled source, as a string, or None if compilation failed.
"""
# User friendly version check.
if not (distutils.version.LooseVersion(_GetJavaVersion()) >=
distutils.version.LooseVersion('1.6')):
logging.error('Requires Java 1.6 or higher. '
'Please visit http://www.java.com/getjava')
return
svn.try_lock(output)
args = ['java', '-jar', os.path.dirname(__file__) + '/lib/yuicompressor-2.4.7.jar', input, '--line-break', '1000',
'--charset', 'gb2312']
if output:
args += ['-o', output]
if flags:
args += flags
command.run(' '.join(args), show_log=True)
return output
def compileSimple(compiler_jar_path, deps, inputs, compiler_flags, roots, exeEos, update_build_version):
minJs, minJsExpired = compile(compiler_jar_path, deps, inputs, compiler_flags, roots[0],
update_build_version=update_build_version)
if minJs:
out = open(minJs, "r")
content = out.read()
content = re.sub('var COMPILED=false;[\s\S]*goog\.scope=function\(fn\)\{fn\.call\(goog\.global\)\};', '',
content)
# 将"goog.require(...);"移除
content = re.sub('goog\.require\([^)]*\);', '', content)
# 将"goog.provide(...);"替换成"CP.util.namespace(...);"
content = re.sub('goog\.provide', 'CP.util.namespace', content)
# 避免CP命名覆盖问题
content = re.sub('var CP=\{(\w+):(\{[^}]*\})\}', 'var CP=CP||{};CP.\1={}', content)
svn.try_lock(minJs)
out = open(minJs, "w")
out.write(content)
minJsExpired = version.updater.get_rel_path(minJsExpired, roots[0])
htmlPaths = version.updater.update(version.updater.get_rel_path(minJs, roots[0]), roots, minJsExpired)
htmlRealPaths = []
paths = [minJs]
for path in htmlPaths:
htmlRealPaths.append(path)
paths.append(path)
if exeEos:
add_eos_mission(paths, minJs, roots[0])
return (minJs, htmlRealPaths)
def write_deps_file(deps, input_namespaces, outputDir, root_with_prefix):
try:
os.makedirs(outputDir)
except:
pass
deps_path = outputDir + "/" + input_namespaces.copy().pop() + ".deps.js"
svn.try_lock(deps_path)
out = open(deps_path, 'w')
out.write('// This file was autogenerated by %s.\n' % sys.argv[0])
out.write('// Please do not edit.\n')
root_with_prefix, prefix = _GetPair(root_with_prefix)
out.writelines(
[_GetDepsLine(js_source.GetPath().replace(root_with_prefix, prefix).replace(os.sep, posixpath.sep), js_source)
for js_source in deps])
logging.info('The deps file : ' + deps_path)
def writeRequires(self, requiresAll):
sourceRemoveComment = self._source
# remove old requires
sourceRemoveComment = source._REQUIRES_REGEX_LINE.sub('', sourceRemoveComment)
sourceRemoveComment = source._PROVIDE_REGEX_LINE.sub('', sourceRemoveComment)
sourceRemoveComment = source.Source._StripComments(sourceRemoveComment)
sourceRemoveComment = source.Source._COMMENT_INLINE_REGEX.sub('', sourceRemoveComment)
sourceRemoveComment = source._STRING_REGEX_LINE.sub('', sourceRemoveComment)
requiresDirect = set()
for moduleName in requiresAll:
if re.search(r'(?<![\w\d\_\-@])' + (moduleName.replace(".", "\.")).replace("$", "\$") + r"(?![\w\d\_\-@])",
sourceRemoveComment):
sourceRemoveComment = sourceRemoveComment.replace(moduleName, self.GetFlatName(moduleName))
requiresDirect.add(moduleName)
requiresDirect.update(self.requires)
for provide in self.provides:
try:
requiresDirect.remove(provide)
except Exception:
pass
# requiresDirect.difference(self.requires)
if requiresDirect.difference(self.requires):
requiresDirect = sorted(requiresDirect)
# update requires
self.requires = requiresDirect
requireStatement = 'goog.require("%s");'
requireStatements = ''
for require in requiresDirect:
requireStatements += (requireStatement % require) + "\n"
requireStatements += "\n"
sourceNew = source._REQUIRES_REGEX_LINE.sub('', self._source)
re_provide = re.compile('((\n?\s*goog\.provide\(\s*[\'"].*[\'"]\s*\);?\n*)+)')
if requiresDirect:
sourceNew = re_provide.sub(r'\1' + requireStatements, sourceNew)
if sourceNew != self._source:
self._source = sourceNew
svn.try_lock(self._path)
out = open(self._path, "w")
out.write(sourceNew)
def compileTemplate(inputs, outputPathFormat):
outputPaths = []
for input in inputs:
output_path = getOutputPath(input, outputPathFormat)
outputPaths.append(output_path)
svn.try_lock(output_path)
args = [
'java', '-jar',
os.path.dirname(__file__) + '/SoyToJsSrcCompiler.jar', '--srcs',
','.join(inputs), '--outputPathFormat', outputPathFormat,
'--shouldProvideRequireSoyNamespaces'
]
logging.info('Compiling with the following command: %s', ' '.join(args))
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
stdoutdata, unused_stderrdata = proc.communicate()
return outputPaths
def compileTemplate(inputs, outputPathFormat):
outputPaths = []
for input in inputs:
output_path = getOutputPath(input, outputPathFormat)
outputPaths.append(output_path)
svn.try_lock(output_path)
args = [
'java',
'-jar', os.path.dirname(__file__) + '/SoyToJsSrcCompiler.jar',
'--srcs', ','.join(inputs),
'--outputPathFormat', outputPathFormat,
'--shouldProvideRequireSoyNamespaces'
]
logging.info('Compiling with the following command: %s', ' '.join(args))
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
stdoutdata, unused_stderrdata = proc.communicate()
return outputPaths
def save_version_to_file(entry_point_file, version):
svn.try_lock(entry_point_file)
file_io = open(entry_point_file, 'r')
content = file_io.read()
file_io.close()
re_compile = re.compile('([\s\S]*)//buildVersion=(.+)')
compile_match = re_compile.match(content)
version_str = '//buildVersion=' + version
if compile_match:
content = re_compile.sub(r'\1' + version_str, content)
else:
content = version_str + '\n' + content
file_io = open(entry_point_file, 'w')
file_io.write(content)
file_io.close()
def save_version_to_file(entry_point_file, version):
svn.try_lock(entry_point_file)
file_io = open(entry_point_file, 'r')
content = file_io.read()
file_io.close()
re_compile = re.compile('([\s\S]*)//buildVersion=(.+)')
compile_match = re_compile.match(content)
version_str = '//buildVersion=' + version
if compile_match:
content = re_compile.sub(r'\1' + version_str, content)
else:
content = version_str + '\n' + content
file_io = open(entry_point_file, 'w')
file_io.write(content)
file_io.close()
def genModuleJsEntryPoint(deps, input_namespaces, modulejs, output_dir, root_with_prefix, host):
# genernate modulejs entrypoint file
entryModule = input_namespaces.pop()
entrypoint_js = entryModule + ".entrypoint.js"
template = open(os.path.dirname(__file__) + '/../../../modulejs/template4entrypoint.js', "r").read()
template = template.replace("/*host*/", host)
rootWithPrefix, prefix = _GetPair(root_with_prefix)
template = template.replace("/*urlMap*/", ',\n'.join(
[_GetModuleJsLine(js_source.GetPath().replace(rootWithPrefix, prefix).replace(os.sep, posixpath.sep), js_source)
for js_source in deps if not 'closure' in js_source.GetPath()]))
template = template.replace("/*moduleJs*/", open(modulejs, "r").read())
template = template.replace("/*entryPointModule*/", entryModule)
dir_entrypoint_js = output_dir + "/" + entrypoint_js
svn.try_lock(dir_entrypoint_js)
out = open(dir_entrypoint_js, 'w')
out.write(template)
logging.info('The entrypoint url is:\nhttp://888.gtimg.com/static/v1.0/i/js/entrypoint/' + entrypoint_js)
logging.info('Success.')
def updateVersionByPathsNew(contentFilePath, regStrArr, fileRelPaths, filePathNewVersion):
result = 0
replacePatten = r'\1' + 'http://888.gtimg.com' + filePathNewVersion + r'\3'
fileObj = open(contentFilePath)
content = fileObj.read()
for fileRelPath in fileRelPaths:
for regStr in regStrArr:
reg = re.compile(regStr % fileRelPath)
if reg.search(content):
content = reg.sub(replacePatten, content)
result = 1
if result:
svn.try_lock(contentFilePath)
fileObjW = open(contentFilePath, "w")
fileObjW.write(content)
fileObjW.close()
fileObj.close()
return result
def updateVersionByPaths(contentFilePath, regStrArr, fileRelPaths, versionStr):
"""Update version
Args:
contentFilePath : str, Path to file.
regStrArr : set, reg pattern set
fileRelPaths : set
versionStr : str
Returns:
int, if found, 0 : not found, 1: found
"""
result = 0
replacePatten = r'\1\2?t=' + versionStr + r'\3'
fileObj = open(contentFilePath)
content = fileObj.read()
for fileRelPath in fileRelPaths:
for regStr in regStrArr:
reg = re.compile(regStr % fileRelPath)
if reg.search(content):
content = reg.sub(replacePatten, content)
result = 1
if result:
svn.try_lock(contentFilePath)
fileObjW = open(contentFilePath, "w")
fileObjW.write(content)
fileObjW.close()
fileObj.close()
return result
def updateVersionByPathsNew(contentFilePath, regStrArr, fileRelPaths,
filePathNewVersion):
result = 0
replacePatten = r'\1' + 'http://888.gtimg.com' + filePathNewVersion + r'\3'
fileObj = open(contentFilePath)
content = fileObj.read()
for fileRelPath in fileRelPaths:
for regStr in regStrArr:
reg = re.compile(regStr % fileRelPath)
if reg.search(content):
content = reg.sub(replacePatten, content)
result = 1
if result:
svn.try_lock(contentFilePath)
fileObjW = open(contentFilePath, "w")
fileObjW.write(content)
fileObjW.close()
fileObj.close()
return result
def updateVersionByPaths(contentFilePath, regStrArr, fileRelPaths, versionStr):
"""Update version
Args:
contentFilePath : str, Path to file.
regStrArr : set, reg pattern set
fileRelPaths : set
versionStr : str
Returns:
int, if found, 0 : not found, 1: found
"""
result = 0
replacePatten = r'\1\2?t=' + versionStr + r'\3'
fileObj = open(contentFilePath)
content = fileObj.read()
for fileRelPath in fileRelPaths:
for regStr in regStrArr:
reg = re.compile(regStr % fileRelPath)
if reg.search(content):
content = reg.sub(replacePatten, content)
result = 1
if result:
svn.try_lock(contentFilePath)
fileObjW = open(contentFilePath, "w")
fileObjW.write(content)
fileObjW.close()
fileObj.close()
return result
def main():
logging.basicConfig(format=(sys.argv[0] + ': %(message)s'),
level=logging.INFO)
options, args = _GetOptionsParser().parse_args()
output_file = options.output_file
roots = options.roots
inputs = options.inputs
namespaces = options.namespaces
output_mode = options.output_mode
output_dir = options.output_dir
root_with_prefix = options.root_with_prefix
compiler_jar_path = options.compiler_jar
compiler_flags = options.compiler_flags
update_build_version = options.update_build_version == 'True'
# Make our output pipe.
if output_file:
svn.try_lock(output_file)
out = open(output_file, 'w')
else:
out = sys.stdout
sources = getSources(roots, args)
# Though deps output doesn't need to query the tree, we still build it
# to validate dependencies.
logging.info('Building dependency tree..')
tree = depstree.DepsTree(sources)
input_namespaces = set()
inputs = inputs or []
for input_path in inputs:
js_input = _GetInputByPath(input_path, sources)
if not js_input:
logging.error('No source matched input %s', input_path)
sys.exit(1)
input_namespaces.update(js_input.provides)
input_namespaces.update(namespaces)
if not input_namespaces:
logging.error('No namespaces found. At least one namespace must be '
'specified with the --namespace or --input flags.')
sys.exit(2)
deps = tree.GetDependencies(input_namespaces)
# The Closure Library base file must go first.
base = _GetClosureBaseFile(sources)
if base:
deps = [base] + deps
if output_mode == 'list':
out.writelines([js_source.GetPath() + '\n' for js_source in deps])
elif output_mode == 'calcdepsIndependent':
write_deps_file(deps, input_namespaces, output_dir, root_with_prefix)
elif output_mode == 'calcdepsIndependentDetail':
for dep in deps:
print dep.provides.copy().pop(), '\t', dep.GetPath(), '\t', len(dep.GetSource())
elif output_mode == 'calcAndOrganizeDepsIndependent':
calcAndOrganizeDepsIndependent(deps, input_namespaces, output_dir, root_with_prefix, sources)
elif output_mode == 'genModuleJsEntryPoint':
modulejs = options.modulejs
genModuleJsEntryPoint(deps, input_namespaces, modulejs, output_dir, root_with_prefix, options.host)
elif output_mode == 'script':
out.writelines([js_source.GetSource() for js_source in deps])
elif output_mode == 'compiled':
min_js, min_js_expired = compile(compiler_jar_path, deps, inputs, compiler_flags, roots[0],
update_build_version=update_build_version)
if min_js:
min_js_expired = version.updater.get_rel_path(min_js_expired, roots[0])
html_paths = version.updater.update(version.updater.get_rel_path(min_js, roots[0]), roots, min_js_expired)
paths = [min_js]
for path in html_paths:
paths.append(path)
if options.eos:
add_eos_mission(paths, min_js, roots[0])
else:
sys.exit(1)
elif output_mode == 'compiledByModule':
namespace_target = input_namespaces.copy().pop()
sources = tree.GetLeafSourcesByNameSpace(namespace_target)
paths = []
for dep in sources:
namespace = dep.provides.copy().pop()
if not re.compile("^test").match(namespace):
min_js, min_js_expired = compile(compiler_jar_path, [base] + tree.GetDependencies(namespace),
[dep.GetPath()], compiler_flags, roots[0],
update_build_version=update_build_version)
if min_js:
min_js_expired = version.updater.get_rel_path(min_js_expired, roots[0])
html_paths = version.updater.update(version.updater.get_rel_path(min_js, roots[0]), roots, min_js_expired)
paths.append(min_js)
for path in html_paths:
paths.append(path)
if options.eos:
add_eos_mission(paths, namespace_target, roots[0])
elif output_mode == 'compiledSimple':
compileSimple(compiler_jar_path, deps, inputs, compiler_flags, roots, options.eos,
update_build_version=update_build_version)
elif output_mode == 'compiledSimpleByModule':
namespace_target = input_namespaces.copy().pop()
sources = tree.GetLeafSourcesByNameSpace(namespace_target)
paths = []
for dep in sources:
namespace = dep.provides.copy().pop()
if not re.compile("^test").match(namespace):
min_js, html_paths = compileSimple(compiler_jar_path, [base] + tree.GetDependencies(namespace),
[dep.GetPath()], compiler_flags, roots, False,
update_build_version=update_build_version)
if min_js:
paths.append(min_js)
for path in html_paths:
paths.append(path)
if options.eos:
add_eos_mission(paths, namespace_target, roots[0])
elif output_mode == 'findEntriesByModule':
sources = tree.GetLeafSourcesByNameSpace(input_namespaces.copy().pop())
print 'length:', str(len(sources))
for dep in sources:
print dep.provides.copy().pop(), '\t', dep.GetPath()
elif output_mode == 'findModulesByModule':
sources = tree.GetDirectSourcesByNameSpace(input_namespaces.copy().pop())
for dep in sources:
print dep.provides.copy().pop(), '\t', dep.GetPath()
elif output_mode == 'calcdepsIndependentByModule':
sources = tree.GetLeafSourcesByNameSpace(input_namespaces.copy().pop())
for dep in sources:
write_deps_file([base] + tree.GetDependencies(dep.provides.copy().pop()), dep.provides, output_dir,
root_with_prefix)
elif output_mode == 'convertToLegosByEntrypoint':
for dep in deps:
convertToLegos(dep)
elif output_mode == 'convertToLegos':
convertToLegos(_PathSource(inputs[0]))
elif output_mode == 'pubDeps2Eos':
abs_paths = []
for dep in deps:
path = dep.GetPath()
abs_paths.append(path)
add_eos_mission(abs_paths, 'deps for : ' + input_namespaces.copy().pop(), roots[0], False)
else:
logging.error('Invalid value for --output flag.')
sys.exit(2)
