def determineRelPathToSdk(appDir, framework_dir, options):
relPath = ''
if sys.platform == 'cygwin':
if re.match( r'^\.{1,2}\/', appDir):
relPath = Path.rel_from_to(normalizePath(appDir), framework_dir)
elif re.match( r'^/cygdrive\b', appDir):
relPath = Path.rel_from_to(appDir, framework_dir)
else:
relPath = Path.rel_from_to(normalizePath(appDir), normalizePath(framework_dir))
else:
relPath = Path.rel_from_to(normalizePath(appDir), normalizePath(framework_dir))
relPath = re.sub(r'\\', "/", relPath)
if relPath[-1] == "/":
relPath = relPath[:-1]
if not os.path.isdir(os.path.join(appDir, relPath)):
console.error("Relative path to qooxdoo directory is not correct: '%s'" % relPath)
sys.exit(1)
if options.type == "contribution":
#relPath = os.path.join(os.pardir, os.pardir, "qooxdoo", QOOXDOO_VERSION)
#relPath = re.sub(r'\\', "/", relPath)
pass
return relPath
def determineRelPathToSdk(appDir, framework_dir, options):
relPath = ''
if sys.platform == 'cygwin':
if re.match(r'^\.{1,2}\/', appDir):
relPath = Path.rel_from_to(normalizePath(appDir), framework_dir)
elif re.match(r'^/cygdrive\b', appDir):
relPath = Path.rel_from_to(appDir, framework_dir)
else:
relPath = Path.rel_from_to(normalizePath(appDir),
normalizePath(framework_dir))
else:
relPath = Path.rel_from_to(normalizePath(appDir),
normalizePath(framework_dir))
relPath = re.sub(r'\\', "/", relPath)
if relPath[-1] == "/":
relPath = relPath[:-1]
if not os.path.isdir(os.path.join(appDir, relPath)):
console.error(
"Relative path to qooxdoo directory is not correct: '%s'" %
relPath)
sys.exit(1)
return relPath
def from_doc_root_to_app_root(jobconf, confObj, doc_root):
japp_root = jobconf.get("compile-options/paths/app-root", "source")
app_root = os.path.normpath(os.path.join(confObj.absPath(japp_root), 'index.html'))
# as soon as app_root and doc_root have a drive letter, the next might fail due to capitalization
_, _, url_path = Path.getCommonPrefix(doc_root, app_root)
url_path = Path.posifyPath(url_path)
return url_path
def from_doc_root_to_app_root(jobconf, confObj, doc_root):
japp_root = jobconf.get("compile-options/paths/app-root", "source")
app_root = os.path.normpath(
os.path.join(confObj.absPath(japp_root), 'index.html'))
# as soon as app_root and doc_root have a drive letter, the next might fail due to capitalization
_, _, url_path = Path.getCommonPrefix(doc_root, app_root)
url_path = Path.posifyPath(url_path)
return url_path
def patchSkeleton(dir, framework_dir, options):
absPath = normalizePath(framework_dir)
if absPath[-1] == "/":
absPath = absPath[:-1]
if sys.platform == 'cygwin':
if re.match( r'^\.{1,2}\/', dir ):
relPath = Path.rel_from_to(normalizePath(dir), framework_dir)
elif re.match( r'^/cygdrive\b', dir):
relPath = Path.rel_from_to(dir, framework_dir)
else:
relPath = Path.rel_from_to(normalizePath(dir), normalizePath(framework_dir))
else:
relPath = os.path.relpath(framework_dir, dir)
#relPath = Path.rel_from_to(normalizePath(dir), normalizePath(framework_dir))
relPath = re.sub(r'\\', "/", relPath)
if relPath[-1] == "/":
relPath = relPath[:-1]
if not os.path.isdir(os.path.join(dir, relPath)):
console.error("Relative path to qooxdoo directory is not correct: '%s'" % relPath)
sys.exit(1)
if options.type == "contribution":
relPath = os.path.join(os.pardir, os.pardir, "qooxdoo", QOOXDOO_VERSION)
relPath = re.sub(r'\\', "/", relPath)
for root, dirs, files in os.walk(dir):
for file in files:
split = file.split(".")
if len(split) >= 3 and split[-2] == "tmpl":
outFile = os.path.join(root, ".".join(split[:-2] + split[-1:]))
inFile = os.path.join(root, file)
console.log("Patching file '%s'" % outFile)
#config = MyTemplate(open(inFile).read())
config = Template(open(inFile).read())
out = open(outFile, "w")
out.write(
config.substitute({
"Name": options.name,
"Namespace": options.namespace,
"NamespacePath" : (options.namespace).replace('.', '/'),
"REL_QOOXDOO_PATH": relPath,
"ABS_QOOXDOO_PATH": absPath,
"QOOXDOO_VERSION": QOOXDOO_VERSION,
"Cache" : options.cache,
}).encode('utf-8')
)
out.close()
os.remove(inFile)
for root, dirs, files in os.walk(dir):
for file in [file for file in files if file.endswith(".py")]:
os.chmod(os.path.join(root, file), (stat.S_IRWXU
|stat.S_IRGRP |stat.S_IXGRP
|stat.S_IROTH |stat.S_IXOTH)) # 0755
def getImageId(imagePath, prefixSpec):
prefix, altprefix = extractFromPrefixSpec(prefixSpec)
imageId = imagePath # init
_, imageId, _ = Path.getCommonPrefix(imagePath, prefix) # assume: imagePath = prefix "/" imageId
if altprefix:
imageId = altprefix + "/" + imageId
imageId = Path.posifyPath(imageId)
return imageId
def getImageId(imagePath, prefixSpec):
prefix, altprefix = extractFromPrefixSpec(prefixSpec)
imageId = imagePath # init
_, imageId, _ = Path.getCommonPrefix(imagePath, prefix) # assume: imagePath = prefix "/" imageId
if altprefix:
imageId = altprefix + "/" + imageId
imageId = Path.posifyPath(imageId)
return imageId
def patchSkeleton(dir, framework_dir, options):
absPath = normalizePath(framework_dir)
if absPath[-1] == "/":
absPath = absPath[:-1]
if sys.platform == 'cygwin':
if re.match( r'^\.{1,2}\/', dir ):
relPath = Path.rel_from_to(normalizePath(dir), framework_dir)
elif re.match( r'^/cygdrive\b', dir):
relPath = Path.rel_from_to(dir, framework_dir)
else:
relPath = Path.rel_from_to(normalizePath(dir), normalizePath(framework_dir))
else:
relPath = Path.rel_from_to(normalizePath(dir), normalizePath(framework_dir))
relPath = re.sub(r'\\', "/", relPath)
if relPath[-1] == "/":
relPath = relPath[:-1]
if not os.path.isdir(os.path.join(dir, relPath)):
console.error("Relative path to qooxdoo directory is not correct: '%s'" % relPath)
sys.exit(1)
if options.type == "contribution":
# TODO: in a final release the following "trunk" would need to be changed
# to an actual version number like "0.8.3"
relPath = os.path.join(os.pardir, os.pardir, "qooxdoo", "0.8.3")
relPath = re.sub(r'\\', "/", relPath)
for root, dirs, files in os.walk(dir):
for file in files:
split = file.split(".")
if len(split) >= 3 and split[1] == "tmpl":
outFile = os.path.join(root, split[0] + "." + ".".join(split[2:]))
inFile = os.path.join(root, file)
console.log("Patching file '%s'" % outFile)
config = Template(open(inFile).read())
out = open(outFile, "w")
out.write(
config.substitute({
"Name": options.name,
"Namespace": options.namespace,
"REL_QOOXDOO_PATH": relPath,
"ABS_QOOXDOO_PATH": absPath,
"QOOXDOO_VERSION": "0.8.3",
"Cache" : options.cache,
}).encode('utf-8')
)
out.close()
os.remove(inFile)
for root, dirs, files in os.walk(dir):
for file in [file for file in files if file.endswith(".py")]:
os.chmod(os.path.join(root, file), (stat.S_IRWXU
|stat.S_IRGRP |stat.S_IXGRP
|stat.S_IROTH |stat.S_IXOTH)) # 0755
def patchSkeleton(dir, framework_dir, options):
absPath = normalizePath(framework_dir)
if absPath[-1] == "/":
absPath = absPath[:-1]
if sys.platform == 'cygwin':
if re.match( r'^\.{1,2}\/', dir ):
relPath = Path.rel_from_to(normalizePath(dir), framework_dir)
elif re.match( r'^/cygdrive\b', dir):
relPath = Path.rel_from_to(dir, framework_dir)
else:
relPath = Path.rel_from_to(normalizePath(dir), normalizePath(framework_dir))
else:
relPath = Path.rel_from_to(normalizePath(dir), normalizePath(framework_dir))
relPath = re.sub(r'\\', "/", relPath)
if relPath[-1] == "/":
relPath = relPath[:-1]
if not os.path.isdir(os.path.join(dir, relPath)):
console.error("Relative path to qooxdoo directory is not correct: '%s'" % relPath)
sys.exit(1)
if options.type == "contribution":
relPath = os.path.join(os.pardir, os.pardir, "qooxdoo", QOOXDOO_VERSION)
relPath = re.sub(r'\\', "/", relPath)
for root, dirs, files in os.walk(dir):
for file in files:
split = file.split(".")
if len(split) >= 3 and split[1] == "tmpl":
outFile = os.path.join(root, split[0] + "." + ".".join(split[2:]))
inFile = os.path.join(root, file)
console.log("Patching file '%s'" % outFile)
config = Template(open(inFile).read())
out = open(outFile, "w")
out.write(
config.substitute({
"Name": options.name,
"Namespace": options.namespace,
"NamespacePath" : (options.namespace).replace('.', '/'),
"REL_QOOXDOO_PATH": relPath,
"ABS_QOOXDOO_PATH": absPath,
"QOOXDOO_VERSION": QOOXDOO_VERSION,
"Cache" : options.cache,
}).encode('utf-8')
)
out.close()
os.remove(inFile)
for root, dirs, files in os.walk(dir):
for file in [file for file in files if file.endswith(".py")]:
os.chmod(os.path.join(root, file), (stat.S_IRWXU
|stat.S_IRGRP |stat.S_IXGRP
|stat.S_IROTH |stat.S_IXOTH)) # 0755
def normalizeImgUri(uriFromMetafile, trueCombinedUri, combinedUriFromMetafile):
# normalize paths (esp. "./x" -> "x")
(uriFromMetafile, trueCombinedUri, combinedUriFromMetafile) = map(os.path.normpath,
(uriFromMetafile, trueCombinedUri, combinedUriFromMetafile))
# get the "wrong" left part of the combined image, as used in the .meta file (in mappedUriPrefix)
trueUriPrefix, mappedUriPrefix, _ = Path.getCommonSuffix(trueCombinedUri, combinedUriFromMetafile)
# ...and strip it from clipped image, to get a correct image id (in uriSuffix)
_, _, uriSuffix = Path.getCommonPrefix(mappedUriPrefix, uriFromMetafile)
# ...then compose the correct prefix with the correct suffix
normalUri = os.path.normpath(os.path.join(trueUriPrefix, uriSuffix))
return normalUri
def patchSkeleton(dir, name, namespace):
absPath = normalizePath(FRAMEWORK_DIR)
if absPath[-1] == "/":
absPath = absPath[:-1]
if sys.platform == 'cygwin':
if re.match(r'^\.{1,2}\/', dir):
relPath = Path.rel_from_to(normalizePath(dir), FRAMEWORK_DIR)
elif re.match(r'^/cygdrive\b', dir):
relPath = Path.rel_from_to(dir, FRAMEWORK_DIR)
else:
relPath = Path.rel_from_to(normalizePath(dir),
normalizePath(FRAMEWORK_DIR))
else:
relPath = Path.rel_from_to(normalizePath(dir),
normalizePath(FRAMEWORK_DIR))
relPath = re.sub(r'\\', "/", relPath)
if relPath[-1] == "/":
relPath = relPath[:-1]
if not os.path.isdir(os.path.join(dir, relPath)):
console.error(
"Relative path to qooxdoo directory is not correct: '%s'" %
relPath)
sys.exit(1)
for root, dirs, files in os.walk(dir):
for file in files:
split = file.split(".")
if len(split) >= 3 and split[1] == "tmpl":
outFile = os.path.join(root,
split[0] + "." + ".".join(split[2:]))
inFile = os.path.join(root, file)
console.log("Patching file '%s'" % outFile)
config = Template(open(inFile).read())
out = open(outFile, "w")
out.write(
config.substitute({
"Name": name,
"Namespace": namespace,
"REL_QOOXDOO_PATH": relPath,
"ABS_QOOXDOO_PATH": absPath,
"QOOXDOO_VERSION": "0.8"
}))
out.close()
os.remove(inFile)
for root, dirs, files in os.walk(dir):
for file in [file for file in files if file.endswith(".py")]:
os.chmod(os.path.join(root, file), 0755)
def patchSkeleton(dir, name, namespace):
absPath = normalizePath(FRAMEWORK_DIR)
if absPath[-1] == "/":
absPath = absPath[:-1]
if sys.platform == 'cygwin':
if re.match( r'^\.{1,2}\/', dir ):
relPath = Path.rel_from_to(normalizePath(dir), FRAMEWORK_DIR)
elif re.match( r'^/cygdrive\b', dir):
relPath = Path.rel_from_to(dir, FRAMEWORK_DIR)
else:
relPath = Path.rel_from_to(normalizePath(dir), normalizePath(FRAMEWORK_DIR))
else:
relPath = Path.rel_from_to(normalizePath(dir), normalizePath(FRAMEWORK_DIR))
relPath = re.sub(r'\\', "/", relPath)
if relPath[-1] == "/":
relPath = relPath[:-1]
if not os.path.isdir(os.path.join(dir, relPath)):
console.error("Relative path to qooxdoo directory is not correct: '%s'" % relPath)
sys.exit(1)
for root, dirs, files in os.walk(dir):
for file in files:
split = file.split(".")
if len(split) >= 3 and split[1] == "tmpl":
outFile = os.path.join(root, split[0] + "." + ".".join(split[2:]))
inFile = os.path.join(root, file)
console.log("Patching file '%s'" % outFile)
config = Template(open(inFile).read())
out = open(outFile, "w")
out.write(
config.substitute({
"Name": name,
"Namespace": namespace,
"REL_QOOXDOO_PATH": relPath,
"ABS_QOOXDOO_PATH": absPath,
"QOOXDOO_VERSION": "0.8.1"
})
)
out.close()
os.remove(inFile)
for root, dirs, files in os.walk(dir):
for file in [file for file in files if file.endswith(".py")]:
os.chmod(os.path.join(root, file), 0755)
def _computeResourceUri(self, lib, resourcePath, rType="class", appRoot=None):
if 'uri' in lib:
libBaseUri = Uri(lib['uri'])
elif appRoot:
libBaseUri = Uri(Path.rel_from_to(self._config.absPath(appRoot), lib['path']))
else:
raise RuntimeError, "Need either lib['uri'] or appRoot, to calculate final URI"
#libBaseUri = Uri(libBaseUri.toUri())
if rType in lib:
libInternalPath = OsPath(lib[rType])
else:
raise RuntimeError, "No such resource type: \"%s\"" % rType
# process the second part of the target uri
uri = libInternalPath.join(resourcePath)
uri = Uri(uri.toUri())
libBaseUri.ensureTrailingSlash()
uri = libBaseUri.join(uri)
# strip dangling "/", e.g. when we have no resourcePath
uri.ensureNoTrailingSlash()
return uri
def _computeResourceUri(self, lib, resourcePath, rType="class", appRoot=None):
'''computes a complete resource URI for the given resource type rType,
from the information given in lib and, if lib doesn't provide a
general uri prefix for it, use appRoot and lib path to construct
one'''
if 'uri' in lib:
libBaseUri = Uri(lib['uri'])
elif appRoot:
libBaseUri = Uri(Path.rel_from_to(self._config.absPath(appRoot), lib['path']))
else:
raise RuntimeError, "Need either lib['uri'] or appRoot, to calculate final URI"
#libBaseUri = Uri(libBaseUri.toUri())
if rType in lib:
libInternalPath = OsPath(lib[rType])
else:
raise RuntimeError, "No such resource type: \"%s\"" % rType
# process the second part of the target uri
uri = libInternalPath.join(resourcePath)
uri = Uri(uri.toUri())
libBaseUri.ensureTrailingSlash()
uri = libBaseUri.join(uri)
return uri
def _computeResourceUri(self, lib, resourcePath, rType="class", appRoot=None):
'''computes a complete resource URI for the given resource type rType,
from the information given in lib and, if lib doesn't provide a
general uri prefix for it, use appRoot and lib path to construct
one'''
if 'uri' in lib:
libBaseUri = Uri(lib['uri'])
elif appRoot:
libBaseUri = Uri(Path.rel_from_to(self._config.absPath(appRoot), lib['path']))
else:
raise RuntimeError, "Need either lib['uri'] or appRoot, to calculate final URI"
#libBaseUri = Uri(libBaseUri.toUri())
if rType in lib:
libInternalPath = OsPath(lib[rType])
else:
raise RuntimeError, "No such resource type: \"%s\"" % rType
# process the second part of the target uri
uri = libInternalPath.join(resourcePath)
uri = Uri(uri.toUri())
libBaseUri.ensureTrailingSlash()
uri = libBaseUri.join(uri)
return uri
def runCopyFiles(jobconf, confObj):
# Copy application files
if not jobconf.get("copy-files/files", False):
return
console = Context.console
filePatts = jobconf.get("copy-files/files", [])
if filePatts:
buildRoot = jobconf.get("copy-files/target", "build")
buildRoot = confObj.absPath(buildRoot)
sourceRoot = jobconf.get("copy-files/source", "source")
sourceRoot = confObj.absPath(sourceRoot)
console.info("Copying application files...")
console.indent()
for patt in filePatts:
appfiles = glob.glob(os.path.join(sourceRoot, patt))
for srcfile in appfiles:
console.debug("copying %s" % srcfile)
srcfileSuffix = Path.getCommonPrefix(sourceRoot, srcfile)[2]
destfile = os.path.join(buildRoot, srcfileSuffix)
if (os.path.isdir(srcfile)):
destdir = destfile
else:
destdir = os.path.dirname(destfile)
_copyResources(srcfile, destdir)
console.outdent()
def _scanResourcePath(self, path):
if not os.path.exists(path):
raise ValueError("The given resource path does not exist: %s" % path)
self._console.debug("Scanning resource folder...")
path = os.path.abspath(path)
lib_prefix_len = len(path)
if not path.endswith(os.sep):
lib_prefix_len += 1
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirectories.match(dir):
dirs.remove(dir)
for file in files:
fpath = os.path.join(root, file)
fpath = os.path.normpath(fpath)
if Image.isImage(fpath):
if CombinedImage.isCombinedImage(fpath):
res = CombinedImage(fpath)
else:
res = Image(fpath)
res.analyzeImage()
else:
res = Resource(fpath)
res.id = Path.posifyPath(fpath[lib_prefix_len:])
res.library= self
self.resources.add(res)
return
def update_helper(char_name, move_nodes):
print('updating moves for', char_name)
char = Database.Characters(char_name)
start = Database.raw_moves[char]
count = 0
if True:
(key_in, key_out) = -1, 0
key_to_val = {i.name: str(i.move_id) for i in move_nodes}
else:
(key_in, key_out) = 0, -1
key_to_val = {str(i.move_id): i.name for i in move_nodes}
for row in start[1:]:
val_in = row[key_in].split(',')
val_out = ','.join(
[j for j in [key_to_val.get(i) for i in val_in] if j])
check_out = row[key_out]
if check_out != val_out:
count += 1
row[key_out] = val_out
print('diff', count)
if count > 0:
csv_content = '\n'.join(['\t'.join(i) for i in start]) + '\n'
path = Path.path('./database/%s.csv' % char.name)
with open(path, 'w', encoding='UTF-8') as fh:
fh.write(csv_content)
print('done')
def filter(respath):
respath = Path.posifyPath(respath)
for res in self._resList:
mo = res.search(respath) # this might need a better 'match' algorithm
if mo:
return True
return False
def runCopyFiles(jobconf, confObj):
# Copy application files
if not jobconf.get("copy-files/files", False):
return
console = Context.console
filePatts = jobconf.get("copy-files/files", [])
if filePatts:
buildRoot = jobconf.get("copy-files/target", "build")
buildRoot = confObj.absPath(buildRoot)
sourceRoot = jobconf.get("copy-files/source", "source")
sourceRoot = confObj.absPath(sourceRoot)
console.info("Copying application files...")
console.indent()
for patt in filePatts:
appfiles = glob.glob(os.path.join(sourceRoot, patt))
for srcfile in appfiles:
console.debug("copying %s" % srcfile)
srcfileSuffix = Path.getCommonPrefix(sourceRoot, srcfile)[2]
destfile = os.path.join(buildRoot, srcfileSuffix)
if os.path.isdir(srcfile):
destdir = destfile
else:
destdir = os.path.dirname(destfile)
_copyResources(srcfile, destdir)
console.outdent()
def generateBootScript(bootPackage=""):
def packagesOfFiles(fileUri, packages):
# returns list of lists, each containing destination file name of the corresp. part
# npackages = [['script/gui-0.js'], ['script/gui-1.js'],...]
npackages = []
file = os.path.basename(fileUri)
for packageId in range(len(packages)):
packageFileName = self._resolveFileName(file, variants, settings, packageId)
npackages.append((packageFileName,))
return npackages
self._console.info("Generating boot script...")
bootBlocks = []
# For resource list
resourceUri = compConf.get('uris/resource', 'resource')
resourceUri = Path.posifyPath(resourceUri)
globalCodes = self.generateGlobalCodes(libs, translationMaps, settings, variants, format, resourceUri, scriptUri)
filesPackages = packagesOfFiles(fileUri, packages)
bootBlocks.append(self.generateBootCode(parts, filesPackages, boot, variants, settings, bootPackage, globalCodes, "build", format))
if format:
bootContent = "\n\n".join(bootBlocks)
else:
bootContent = "".join(bootBlocks)
return bootContent
def getSavePath(self, urlRoot, fileRoot, currUrl):
savePath = pathFrag = ""
# pathFrag = currUrl - urlRoot; assert currUrl > urlRoot
assert currUrl.startswith(urlRoot) # only consider urls from the same root
(pre,sfx1,sfx2) = Path.getCommonPrefix(urlRoot, currUrl)
pathFrag = sfx2
savePath = os.path.join(fileRoot, pathFrag)
return savePath
def assetIdFromPath(self, resource, lib):
def extractAssetPart(libresuri, imguri):
pre,libsfx,imgsfx = Path.getCommonPrefix(libresuri, imguri) # split libresuri from imguri
if imgsfx[0] == os.sep: imgsfx = imgsfx[1:] # strip leading '/'
return imgsfx # use the bare img suffix as its asset Id
librespath = os.path.normpath(os.path.join(lib['path'], lib['resource']))
assetId = extractAssetPart(librespath, resource)
assetId = Path.posifyPath(assetId)
return assetId
def getSavePath(self, urlRoot, fileRoot, currUrl):
savePath = pathFrag = ""
# pathFrag = currUrl - urlRoot; assert currUrl > urlRoot
assert currUrl.startswith(
urlRoot) # only consider urls from the same root
(pre, sfx1, sfx2) = Path.getCommonPrefix(urlRoot, currUrl)
pathFrag = sfx2
savePath = os.path.join(fileRoot, pathFrag)
return savePath
def resourceValue(r):
# create a pair res = [path, uri] for this resource...
rsource = os.path.normpath(r) # normalize "./..."
relpath = (Path.getCommonPrefix(libObj._resourcePath, rsource))[2]
if relpath[0] == os.sep: # normalize "/..."
relpath = relpath[1:]
ruri = (self._genobj._computeResourceUri(lib, relpath, rType='resource',
appRoot=self._genobj.approot))
return (rsource, ruri)
def populate_character(character_name):
existing = [i for i in database.keys() if i[0] == character_name]
for i in existing:
del database[i]
path = Path.path('./database/%s.csv' % character_name)
with open(path, encoding='UTF-8') as csvfile:
reader = csv.reader(csvfile, delimiter='\t')
data = [i for i in reader if i]
raw_moves[Characters[character_name]] = data
header = data[0]
for move in data[1:]:
populate_move(character_name, header, move)
def init_tk(self):
self.wm_title("dcep93/TekkenBot")
self.iconbitmap(Path.path('./img/tekken_bot_close.ico'))
self.text = t_tkinter.Text(self, wrap="word")
stdout = sys.stdout
sys.stdout = TextRedirector(self.text, stdout, "stdout")
stderr = sys.stderr
sys.stderr = TextRedirector(self.text, stderr, "stderr")
self.text.tag_configure("stderr", foreground="#b22222")
self.text.pack(fill=t_tkinter.BOTH)
self.geometry('1200x420+0+0')
def _scanResourcePath(self):
resources = set()
if self.resourcePath is None:
return resources
if not os.path.isdir(os.path.join(self.path, self.resourcePath)):
self._console.info("Lib<%s>: Skipping non-existing resource path" %
self.namespace)
return resources
path = os.path.join(self.path, self.resourcePath)
self._console.debug("Scanning resource folder...")
path = os.path.abspath(path)
lib_prefix_len = len(path)
if not path.endswith(os.sep):
lib_prefix_len += 1
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirEntries.match(dir):
dirs.remove(dir)
for file in files:
if self._ignoredDirEntries.match(file):
continue
fpath = os.path.join(root, file)
fpath = os.path.normpath(fpath)
if Image.isImage(fpath):
if CombinedImage.isCombinedImage(fpath):
res = CombinedImage(fpath)
else:
res = Image(fpath)
res.analyzeImage()
elif FontMap.isFontMap(fpath):
res = FontMap(fpath)
else:
res = Resource(fpath)
res.set_id(Path.posifyPath(fpath[lib_prefix_len:]))
res.library = self
resources.add(res)
self._console.indent()
self._console.debug("Found %s resources" % len(resources))
self._console.outdent()
return resources
def _scanResourcePath(self):
resources = set()
if self.resourcePath is None:
return resources
if not os.path.isdir(os.path.join(self.path,self.resourcePath)):
self._console.info("Lib<%s>: Skipping non-existing resource path" % self.namespace)
return resources
path = os.path.join(self.path,self.resourcePath)
self._console.debug("Scanning resource folder...")
path = os.path.abspath(path)
lib_prefix_len = len(path)
if not path.endswith(os.sep):
lib_prefix_len += 1
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirEntries.match(dir):
dirs.remove(dir)
for file in files:
if self._ignoredDirEntries.match(file):
continue
fpath = os.path.join(root, file)
fpath = os.path.normpath(fpath)
if Image.isImage(fpath):
if CombinedImage.isCombinedImage(fpath):
res = CombinedImage(fpath)
else:
res = Image(fpath)
res.analyzeImage()
elif FontMap.isFontMap(fpath):
res = FontMap(fpath)
else:
res = Resource(fpath)
res.set_id(Path.posifyPath(fpath[lib_prefix_len:]))
res.library= self
resources.add(res)
self._console.indent()
self._console.debug("Found %s resources" % len(resources))
self._console.outdent()
return resources
def findLibResources(self, lib, filter=None):
def getCache(lib):
cacheId = "resinlib-%s" % lib._path
liblist = self._genobj._cache.read(cacheId, dependsOn=None, memory=True)
return liblist, cacheId
def isSkipFile(f):
if [x for x in map(lambda x: re.search(x, f), ignoredFiles) if x!=None]:
return True
else:
return False
# - Main --------------------------------------------------------------
cacheList = [] # to poss. populate cache
cacheId = "" # will be filled in getCache()
ignoredFiles = [r'\.meta$',] # files not considered as resources
# create wrapper object
libObj = Library(lib, self._genobj._console)
# retrieve list of library resources
libList, cacheId = getCache(libObj)
if libList:
inCache = True
else:
libList = libObj.scanResourcePath()
inCache = False
# go through list of library resources and add suitable
for resource in libList:
# scanResourcePath() yields absolute paths to a resource, but
# we only want to match against the 'resource' part of it
resourcePart = Path.getCommonPrefix(libObj._resourcePath, resource)[2]
if not inCache:
cacheList.append(resource)
if isSkipFile(resource):
continue
elif (filter and not filter(resourcePart)):
continue
else:
yield resource
if not inCache:
# cache write
self._genobj._cache.write(cacheId, cacheList, memory=True, writeToFile=False)
return
def analyseResource(resource, lib):
##
# compute the resource value of an image for the script
def imgResVal(resource):
imgId = resource
# Cache or generate
if (imgId in imgLookupTable and
imgLookupTable[imgId ]["time"] > os.stat(imgId ).st_mtime):
imageInfo = imgLookupTable[imgId ]["content"]
else:
imageInfo = self._imageInfo.getImageInfo(imgId , assetId)
imgLookupTable[imgId ] = {"time": time.time(), "content": imageInfo}
# Now process the image information
# use an ImgInfoFmt object, to abstract from flat format
imgfmt = ImgInfoFmt()
imgfmt.lib = lib['namespace']
if not 'type' in imageInfo:
raise RuntimeError, "Unable to get image info from file: %s" % imgId
imgfmt.type = imageInfo['type']
# Add this image
# imageInfo = {width, height, filetype}
if not 'width' in imageInfo or not 'height' in imageInfo:
raise RuntimeError, "Unable to get image info from file: %s" % imgId
imgfmt.width = imageInfo['width']
imgfmt.height = imageInfo['height']
imgfmt.type = imageInfo['type']
return imgfmt
# ----------------------------------------------------------
imgpatt = re.compile(r'\.(png|jpeg|jpg|gif)$', re.I)
librespath = os.path.normpath(os.path.join(lib['path'], lib['resource']))
assetId = extractAssetPart(librespath, resource)
assetId = Path.posifyPath(assetId)
if imgpatt.search(resource): # handle images
resvalue = imgResVal(resource)
else: # handle other resources
resvalue = lib['namespace']
return assetId, resvalue
def _scanResourcePath(self, path):
if not os.path.exists(path):
raise ValueError("The given resource path does not exist: %s" %
path)
self._console.debug("Scanning resource folder...")
path = os.path.abspath(path)
lib_prefix_len = len(path)
if not path.endswith(os.sep):
lib_prefix_len += 1
self.resources = set()
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirEntries.match(dir):
dirs.remove(dir)
for file in files:
if self._ignoredDirEntries.match(file):
continue
fpath = os.path.join(root, file)
fpath = os.path.normpath(fpath)
if Image.isImage(fpath):
if CombinedImage.isCombinedImage(fpath):
res = CombinedImage(fpath)
else:
res = Image(fpath)
res.analyzeImage()
else:
res = Resource(fpath)
res.set_id(Path.posifyPath(fpath[lib_prefix_len:]))
res.library = self
self.resources.add(res)
self._console.indent()
self._console.debug("Found %s resources" % len(self.resources))
self._console.outdent()
return
def analyseResource(self, resource):
##
# compute the resource value of an image for the script
def imgResVal(resource, assetId):
imageInfo = self._imageInfoObj.getImageInfo(resource, assetId)
# Now process the image information
# use an ImgInfoFmt object, to abstract from flat format
imgfmt = ImgInfoFmt()
imgfmt.lib = self.namespace
if not "type" in imageInfo:
raise RuntimeError, "Unable to get image info from file: %s" % resource
imgfmt.type = imageInfo["type"]
# Add this image
# imageInfo = {width, height, filetype}
if not "width" in imageInfo or not "height" in imageInfo:
raise RuntimeError, "Unable to get image info from file: %s" % resource
imgfmt.width = imageInfo["width"]
imgfmt.height = imageInfo["height"]
imgfmt.type = imageInfo["type"]
return imgfmt
# ----------------------------------------------------------
imgpatt = re.compile(r"\.(png|jpeg|jpg|gif)$", re.I)
librespath = os.path.normpath(self._resourcePath)
lib_prefix_len = len(librespath)
if not librespath.endswith(os.sep):
lib_prefix_len += 1
assetId = resource[lib_prefix_len:]
assetId = Path.posifyPath(assetId)
if imgpatt.search(resource): # handle images
resvalue = imgResVal(resource, assetId)
else: # handle other resources
resvalue = self.namespace
return assetId, resvalue
def processCombinedImg(data, meta_fname, combinedImageUri, combinedImageShortUri, combinedImageObject):
# make sure lib and type info for the combined image are present
assert combinedImageObject.lib, combinedImageObject.type
# see if we have cached the contents (json) of this .meta file
cacheId = "imgcomb-%s" % meta_fname
imgDict = self._cache.read(cacheId, meta_fname)
if imgDict == None:
mfile = open(meta_fname)
imgDict = simplejson.loads(mfile.read())
mfile.close()
self._cache.write(cacheId, imgDict)
# now loop through the dict structure from the .meta file
for imagePath, imageSpec_ in imgDict.items():
# sort of like this: imagePath : [width, height, type, combinedUri, off-x, off-y]
imageObject = ImgInfoFmt(imageSpec_) # turn this into an ImgInfoFmt object, to abstract from representation in .meta file and loader script
# have to normalize the uri's from the meta file
#imageUri = normalizeImgUri(imagePath, combinedImageUri, imageObject.mappedId)
imageUri = imagePath
## replace lib uri with lib namespace in imageUri
imageShortUri = extractAssetPart(librespath, imageUri)
imageShortUri = Path.posifyPath(imageShortUri)
# now put all elements of the image object together
imageObject.mappedId = combinedImageShortUri # correct the mapped uri of the combined image
imageObject.lib = combinedImageObject.lib
imageObject.mtype = combinedImageObject.type
imageObject.mlib = combinedImageObject.lib
# and store it in the data structure
data[imageShortUri] = imageObject.flatten() # this information takes precedence over existing
return
def analyseResource(self, resource):
##
# compute the resource value of an image for the script
def imgResVal(resource, assetId):
imageInfo = self._imageInfoObj.getImageInfo(resource, assetId)
# Now process the image information
# use an ImgInfoFmt object, to abstract from flat format
imgfmt = ImgInfoFmt()
imgfmt.lib = self.namespace
if not 'type' in imageInfo:
raise RuntimeError, "Unable to get image info from file: %s" % resource
imgfmt.type = imageInfo['type']
# Add this image
# imageInfo = {width, height, filetype}
if not 'width' in imageInfo or not 'height' in imageInfo:
raise RuntimeError, "Unable to get image info from file: %s" % resource
imgfmt.width = imageInfo['width']
imgfmt.height = imageInfo['height']
imgfmt.type = imageInfo['type']
return imgfmt
# ----------------------------------------------------------
imgpatt = re.compile(r'\.(png|jpeg|jpg|gif)$', re.I)
librespath = os.path.normpath(self._resourcePath)
lib_prefix_len = len(librespath)
if not librespath.endswith(os.sep):
lib_prefix_len += 1
assetId = resource[lib_prefix_len:]
assetId = Path.posifyPath(assetId)
if imgpatt.search(resource): # handle images
resvalue = imgResVal(resource, assetId)
else: # handle other resources
resvalue = self.namespace
return assetId, resvalue
def _computeResourceUri(self, lib_, resourcePath, rType="class", appRoot=None):
# i still use dict-type lib representation to create _output_ pseudo-lib
if isinstance(lib_, generator.resource.Library.Library):
lib = {
'class' : lib_.classPath,
'uri' : lib_.uri,
'path' : lib_.path,
'resource' : lib_.resourcePath,
}
else:
lib = lib_
if 'uri' in lib and lib['uri'] is not None:
libBaseUri = Uri(lib['uri'])
elif appRoot:
libBaseUri = Uri(Path.rel_from_to(self._config.absPath(appRoot), lib['path']))
else:
raise RuntimeError, "Need either lib.uri or appRoot, to calculate final URI"
#libBaseUri = Uri(libBaseUri.toUri())
if rType in lib:
libInternalPath = OsPath(lib[rType])
else:
raise RuntimeError, "No such resource type: \"%s\"" % rType
# process the second part of the target uri
uri = libInternalPath.join(resourcePath)
uri = Uri(uri.toUri())
libBaseUri.ensureTrailingSlash()
uri = libBaseUri.join(uri)
# strip dangling "/", e.g. when we have no resourcePath
uri.ensureNoTrailingSlash()
return uri
def runCompiled(self, script, treeCompiler, version="build"):
def getOutputFile(compileType):
filePath = compConf.get("paths/file")
if not filePath:
filePath = os.path.join(compileType, "script", script.namespace + ".js")
return filePath
def getFileUri(scriptUri):
appfile = os.path.basename(fileRelPath)
fileUri = os.path.join(scriptUri, appfile) # make complete with file name
fileUri = Path.posifyPath(fileUri)
return fileUri
##
# returns the Javascript code for the initial ("boot") script as a string,
# using the loader.tmpl template and filling its placeholders
def generateBootCode(parts, packages, boot, script, compConf, variants, settings, bootCode, globalCodes, version="source", decodeUrisFile=None, format=False):
##
# create a map with part names as key and array of package id's and
# return as string
def partsMap(script):
partData = {}
packages = script.packagesSortedSimple()
#print "packages: %r" % packages
for part in script.parts:
partData[part] = script.parts[part].packagesAsIndices(packages)
#print "part '%s': %r" % (part, script.parts[part].packages)
partData = json.dumpsCode(partData)
return partData
def fillTemplate(vals, template):
# Fill the code template with various vals
templ = MyTemplate(template)
result = templ.safe_substitute(vals)
return result
def packageUrisToJS1(packages, version, namespace=None):
# Translate URI data to JavaScript
allUris = []
for packageId, package in enumerate(packages):
packageUris = []
for fileId in package:
if version == "build":
# TODO: gosh, the next is an ugly hack!
#namespace = self._resourceHandler._genobj._namespaces[0] # all name spaces point to the same paths in the libinfo struct, so any of them will do
if not namespace:
namespace = script.namespace # all name spaces point to the same paths in the libinfo struct, so any of them will do
relpath = OsPath(fileId)
else:
namespace = self._classes[fileId]["namespace"]
relpath = OsPath(self._classes[fileId]["relpath"])
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
allUris.append(packageUris)
return allUris
##
# Translate URI data to JavaScript
# using Package objects
def packageUrisToJS(packages, version):
allUris = []
for packageId, package in enumerate(packages):
packageUris = []
if package.file: # build
namespace = "__out__"
fileId = package.file
relpath = OsPath(fileId)
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
else: # "source" :
for clazz in package.classes:
namespace = self._classes[clazz]["namespace"]
relpath = OsPath(self._classes[clazz]["relpath"])
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
allUris.append(packageUris)
return allUris
def loadTemplate(bootCode):
# try custom loader templates
loaderFile = compConf.get("paths/loader-template", None)
if not loaderFile:
# use default templates
if version=="build":
#loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader-build.tmpl.js")
# TODO: test-wise using generic template
loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader.tmpl.js")
else:
#loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader-source.tmpl.js")
loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader.tmpl.js")
template = filetool.read(loaderFile)
return template
# ---------------------------------------------------------------
if not parts:
return ""
result = ""
vals = {}
packages = script.packagesSortedSimple()
loader_with_boot = self._job.get("packages/loader-with-boot", True)
# stringify data in globalCodes
for entry in globalCodes:
globalCodes[entry] = json.dumpsCode(globalCodes[entry])
# undo damage done by simplejson to raw strings with escapes \\ -> \
globalCodes[entry] = globalCodes[entry].replace('\\\\\\', '\\').replace(r'\\', '\\') # " gets tripple escaped, therefore the first .replace()
vals.update(globalCodes)
if version=="build":
vals["Resources"] = json.dumpsCode({}) # TODO: undo Resources from globalCodes!!!
vals["Boot"] = '"%s"' % boot
if version == "build":
vals["BootPart"] = bootCode
else:
vals["BootPart"] = ""
# fake package data
for key, package in enumerate(packages):
vals["BootPart"] += "qx.$$packageData['%d']={};\n" % key
# Translate part information to JavaScript
vals["Parts"] = partsMap(script)
# Translate URI data to JavaScript
#vals["Uris"] = packageUrisToJS1(packages, version)
vals["Uris"] = packageUrisToJS(packages, version)
vals["Uris"] = json.dumpsCode(vals["Uris"])
# Add potential extra scripts
vals["UrisBefore"] = []
if self._job.get("add-script", False):
additional_scripts = self._job.get("add-script",[])
for additional_script in additional_scripts:
vals["UrisBefore"].append(additional_script["uri"])
vals["UrisBefore"] = json.dumpsCode(vals["UrisBefore"])
# Whether boot package is inline
if version == "source":
vals["BootIsInline"] = json.dumpsCode(False)
else:
vals["BootIsInline"] = json.dumpsCode(loader_with_boot)
# Closure package information
cParts = {}
if version == "build":
for part in script.parts:
if not loader_with_boot or part != "boot":
cParts[part] = True
vals["ClosureParts"] = json.dumpsCode(cParts)
# Package Hashes
vals["PackageHashes"] = {}
for key, package in enumerate(packages):
if package.hash:
vals["PackageHashes"][key] = package.hash
else:
vals["PackageHashes"][key] = "%d" % key # fake code package hashes in source ver.
vals["PackageHashes"] = json.dumpsCode(vals["PackageHashes"])
# Script hook for qx.$$loader.decodeUris() function
vals["DecodeUrisPlug"] = ""
if decodeUrisFile:
plugCode = filetool.read(self._config.absPath(decodeUrisFile)) # let it bomb if file can't be read
vals["DecodeUrisPlug"] = plugCode.strip()
# Enable "?nocache=...." for script loading?
vals["NoCacheParam"] = "true" if self._job.get("compile-options/uris/add-nocache-param", True) else "false"
# Locate and load loader basic script
template = loadTemplate(bootCode)
# Fill template gives result
result = fillTemplate(vals, template)
return result
##
# shallow layer above generateBootCode(), and its only client
def generateBootScript(globalCodes, script, bootPackage="", compileType="build"):
self._console.info("Generating boot script...")
if not self._job.get("packages/i18n-with-boot", True):
# remove I18N info from globalCodes, so they don't go into the loader
globalCodes["Translations"] = {}
globalCodes["Locales"] = {}
else:
if compileType == "build":
# also remove them here, as this info is now with the packages
globalCodes["Translations"] = {}
globalCodes["Locales"] = {}
plugCodeFile = compConf.get("code/decode-uris-plug", False)
if compileType == "build":
filepackages = [(x.file,) for x in packages]
bootContent = generateBootCode(parts, filepackages, boot, script, compConf, variants, settings, bootPackage, globalCodes, compileType, plugCodeFile, format)
else:
filepackages = [x.classes for x in packages]
bootContent = generateBootCode(parts, filepackages, boot, script, compConf, variants={}, settings={}, bootCode=None, globalCodes=globalCodes, version=compileType, decodeUrisFile=plugCodeFile, format=format)
return bootContent
def getPackageData(package):
data = {}
data["resources"] = package.data.resources
data["translations"] = package.data.translations
data["locales"] = package.data.locales
data = json.dumpsCode(data)
data += ';\n'
return data
def compilePackage(packageIndex, package):
self._console.info("Compiling package #%s:" % packageIndex, False)
self._console.indent()
# Compile file content
pkgCode = self._treeCompiler.compileClasses(package.classes, variants, optimize, format)
pkgData = getPackageData(package)
hash = sha.getHash(pkgData + pkgCode)[:12] # first 12 chars should be enough
isBootPackage = packageIndex == 0
if isBootPackage:
compiledContent = ("qx.$$packageData['%s']=" % hash) + pkgData + pkgCode
else:
compiledContent = u'''qx.$$packageData['%s']=%s\n''' % (hash, pkgData)
compiledContent += u'''qx.Part.$$notifyLoad("%s", function() {\n%s\n});''' % (hash, pkgCode)
#
package.hash = hash # to fill qx.$$loader.packageHashes in generateBootScript()
self._console.debug("Done: %s" % self._computeContentSize(compiledContent))
self._console.outdent()
return compiledContent
##
# takes an array of (po-data, locale-data) dict pairs
# merge all po data and all cldr data in a single dict each
def mergeTranslationMaps(transMaps):
poData = {}
cldrData = {}
for pac_dat, loc_dat in transMaps:
for loc in pac_dat:
if loc not in poData:
poData[loc] = {}
poData[loc].update(pac_dat[loc])
for loc in loc_dat:
if loc not in cldrData:
cldrData[loc] = {}
cldrData[loc].update(loc_dat[loc])
return (poData, cldrData)
# -- Main - runCompiled ------------------------------------------------
# Early return
compileType = self._job.get("compile/type", "")
if compileType not in ("build", "source"):
return
packages = script.packagesSortedSimple()
parts = script.parts
boot = script.boot
variants = script.variants
libraries = script.libraries
self._treeCompiler = treeCompiler
self._variants = variants
self._script = script
self._console.info("Generate %s version..." % compileType)
self._console.indent()
# - Evaluate job config ---------------------
# Compile config
compConf = self._job.get("compile-options")
compConf = ExtMap(compConf)
# Whether the code should be formatted
format = compConf.get("code/format", False)
script.scriptCompress = compConf.get("paths/gzip", False)
# Read in settings
settings = self.getSettings()
script.settings = settings
# Read libraries
libs = self._job.get("library", [])
# Get translation maps
locales = compConf.get("code/locales", [])
translationMaps = self.getTranslationMaps(packages, variants, locales)
# Read in base file name
fileRelPath = getOutputFile(compileType)
filePath = self._config.absPath(fileRelPath)
script.baseScriptPath = filePath
if compileType == "build":
# read in uri prefixes
scriptUri = compConf.get('uris/script', 'script')
scriptUri = Path.posifyPath(scriptUri)
fileUri = getFileUri(scriptUri)
# for resource list
resourceUri = compConf.get('uris/resource', 'resource')
resourceUri = Path.posifyPath(resourceUri)
else:
# source version needs place where the app HTML ("index.html") lives
self.approot = self._config.absPath(compConf.get("paths/app-root", ""))
resourceUri = None
scriptUri = None
# Get global script data (like qxlibraries, qxresources,...)
globalCodes = {}
globalCodes["Settings"] = settings
globalCodes["Variants"] = self.generateVariantsCode(variants)
globalCodes["Libinfo"] = self.generateLibInfoCode(libs, format, resourceUri, scriptUri)
# add synthetic output lib
if scriptUri: out_sourceUri= scriptUri
else:
out_sourceUri = self._computeResourceUri({'class': ".", 'path': os.path.dirname(script.baseScriptPath)}, OsPath(""), rType="class", appRoot=self.approot)
out_sourceUri = os.path.normpath(out_sourceUri.encodedValue())
globalCodes["Libinfo"]['__out__'] = { 'sourceUri': out_sourceUri }
globalCodes["Resources"] = self.generateResourceInfoCode(script, settings, libraries, format)
globalCodes["Translations"],\
globalCodes["Locales"] = mergeTranslationMaps(translationMaps)
# Potentally create dedicated I18N packages
i18n_as_parts = not self._job.get("packages/i18n-with-boot", True)
if i18n_as_parts:
script = self.generateI18NParts(script, globalCodes)
self.writePackages([p for p in script.packages if getattr(p, "__localeflag", False)], script)
if compileType == "build":
# - Specific job config ---------------------
# read in compiler options
optimize = compConf.get("code/optimize", [])
self._treeCompiler.setOptimize(optimize)
# - Generating packages ---------------------
self._console.info("Generating packages...")
self._console.indent()
bootPackage = ""
for packageIndex, package in enumerate(packages):
package.compiled = compilePackage(packageIndex, package)
self._console.outdent()
if not len(packages):
raise RuntimeError("No valid boot package generated.")
# - Put loader and packages together -------
loader_with_boot = self._job.get("packages/loader-with-boot", True)
# handle loader and boot package
if not loader_with_boot:
loadPackage = Package(0) # make a dummy Package for the loader
packages.insert(0, loadPackage)
# attach file names (do this before calling generateBootScript)
for package, fileName in zip(packages, self.packagesFileNames(script.baseScriptPath, len(packages))):
package.file = os.path.basename(fileName)
if self._job.get("compile-options/paths/scripts-add-hash", False):
package.file = self._fileNameWithHash(package.file, package.hash)
# generate and integrate boot code
if loader_with_boot:
# merge loader code with first package
bootCode = generateBootScript(globalCodes, script, packages[0].compiled)
packages[0].compiled = bootCode
else:
loaderCode = generateBootScript(globalCodes, script)
packages[0].compiled = loaderCode
# write packages
self.writePackages(packages, script)
# ---- 'source' version ------------------------------------------------
else:
sourceContent = generateBootScript(globalCodes, script, bootPackage="", compileType=compileType)
# Construct file name
resolvedFilePath = self._resolveFileName(filePath, variants, settings)
# Save result file
filetool.save(resolvedFilePath, sourceContent)
if compConf.get("paths/gzip"):
filetool.gzip(resolvedFilePath, sourceContent)
self._console.outdent()
self._console.debug("Done: %s" % self._computeContentSize(sourceContent))
self._console.outdent()
self._console.outdent()
return # runCompiled()
def getFileUri(scriptUri):
appfile = os.path.basename(fileRelPath)
fileUri = os.path.join(scriptUri, appfile) # make complete with file name
fileUri = Path.posifyPath(fileUri)
return fileUri
def get_path(file_name=RAW_PATH):
return Path.path('./record/%s' % file_name)
def runCompiled(self, script, treeCompiler, version="build"):
def getOutputFile(compileType):
filePath = compConf.get("paths/file")
if not filePath:
filePath = os.path.join(compileType, "script", script.namespace + ".js")
return filePath
def getFileUri(scriptUri):
appfile = os.path.basename(fileRelPath)
fileUri = os.path.join(scriptUri, appfile) # make complete with file name
fileUri = Path.posifyPath(fileUri)
return fileUri
##
# returns the Javascript code for the initial ("boot") script as a string,
# using the loader.tmpl template and filling its placeholders
def generateBootCode(parts, packages, boot, script, compConf, variants, settings, bootCode, globalCodes, version="source", decodeUrisFile=None, format=False):
##
# create a map with part names as key and array of package id's and
# return as string
def partsMap(script):
partData = {}
packages = script.packagesSortedSimple()
#print "packages: %r" % packages
for part in script.parts:
partData[part] = script.parts[part].packagesAsIndices(packages)
#print "part '%s': %r" % (part, script.parts[part].packages)
partData = json.dumpsCode(partData)
return partData
def fillTemplate(vals, template):
# Fill the code template with various vals
templ = MyTemplate(template)
result = templ.safe_substitute(vals)
return result
def packageUrisToJS1(packages, version, namespace=None):
# Translate URI data to JavaScript
allUris = []
for packageId, package in enumerate(packages):
packageUris = []
for fileId in package:
if version == "build":
# TODO: gosh, the next is an ugly hack!
#namespace = self._resourceHandler._genobj._namespaces[0] # all name spaces point to the same paths in the libinfo struct, so any of them will do
if not namespace:
namespace = script.namespace # all name spaces point to the same paths in the libinfo struct, so any of them will do
relpath = OsPath(fileId)
else:
namespace = self._classes[fileId]["namespace"]
relpath = OsPath(self._classes[fileId]["relpath"])
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
allUris.append(packageUris)
return allUris
##
# Translate URI data to JavaScript
# using Package objects
def packageUrisToJS(packages, version):
allUris = []
for packageId, package in enumerate(packages):
packageUris = []
if package.file: # build
namespace = "__out__"
fileId = package.file
relpath = OsPath(fileId)
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
else: # "source" :
for clazz in package.classes:
namespace = self._classes[clazz]["namespace"]
relpath = OsPath(self._classes[clazz]["relpath"])
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
allUris.append(packageUris)
return allUris
def loadTemplate(bootCode):
# try custom loader templates
loaderFile = compConf.get("paths/loader-template", None)
if not loaderFile:
# use default templates
if version=="build":
#loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader-build.tmpl.js")
# TODO: test-wise using generic template
loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader.tmpl.js")
else:
#loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader-source.tmpl.js")
loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader.tmpl.js")
template = filetool.read(loaderFile)
return template
# ---------------------------------------------------------------
if not parts:
return ""
result = ""
vals = {}
packages = script.packagesSortedSimple()
loader_with_boot = self._job.get("packages/loader-with-boot", True)
# stringify data in globalCodes
for entry in globalCodes:
globalCodes[entry] = json.dumpsCode(globalCodes[entry])
# undo damage done by simplejson to raw strings with escapes \\ -> \
globalCodes[entry] = globalCodes[entry].replace('\\\\\\', '\\').replace(r'\\', '\\') # " gets tripple escaped, therefore the first .replace()
vals.update(globalCodes)
if version=="build":
vals["Resources"] = json.dumpsCode({}) # TODO: undo Resources from globalCodes!!!
vals["Boot"] = '"%s"' % boot
if version == "build":
vals["BootPart"] = bootCode
else:
vals["BootPart"] = ""
# fake package data
for key, package in enumerate(packages):
vals["BootPart"] += "qx.$$packageData['%d']={};\n" % key
# Translate part information to JavaScript
vals["Parts"] = partsMap(script)
# Translate URI data to JavaScript
#vals["Uris"] = packageUrisToJS1(packages, version)
vals["Uris"] = packageUrisToJS(packages, version)
vals["Uris"] = json.dumpsCode(vals["Uris"])
# Add potential extra scripts
vals["UrisBefore"] = []
if self._job.get("add-script", False):
additional_scripts = self._job.get("add-script",[])
for additional_script in additional_scripts:
vals["UrisBefore"].append(additional_script["uri"])
vals["UrisBefore"] = json.dumpsCode(vals["UrisBefore"])
# Whether boot package is inline
if version == "source":
vals["BootIsInline"] = json.dumpsCode(False)
else:
vals["BootIsInline"] = json.dumpsCode(loader_with_boot)
# Closure package information
cParts = {}
if version == "build":
for part in script.parts:
if not loader_with_boot or part != "boot":
cParts[part] = True
vals["ClosureParts"] = json.dumpsCode(cParts)
# Package Hashes
vals["PackageHashes"] = {}
for key, package in enumerate(packages):
if package.hash:
vals["PackageHashes"][key] = package.hash
else:
vals["PackageHashes"][key] = "%d" % key # fake code package hashes in source ver.
vals["PackageHashes"] = json.dumpsCode(vals["PackageHashes"])
# Script hook for qx.$$loader.decodeUris() function
vals["DecodeUrisPlug"] = ""
if decodeUrisFile:
plugCode = filetool.read(self._config.absPath(decodeUrisFile)) # let it bomb if file can't be read
vals["DecodeUrisPlug"] = plugCode.strip()
# Enable "?nocache=...." for script loading?
vals["NoCacheParam"] = "true" if self._job.get("compile-options/uris/add-nocache-param", True) else "false"
# Add build details
vals["Build"] = int(time.time()*1000)
vals["Type"] = version
# Locate and load loader basic script
template = loadTemplate(bootCode)
# Fill template gives result
result = fillTemplate(vals, template)
return result
##
# shallow layer above generateBootCode(), and its only client
def generateBootScript(globalCodes, script, bootPackage="", compileType="build"):
self._console.info("Generating boot script...")
if not self._job.get("packages/i18n-with-boot", True):
# remove I18N info from globalCodes, so they don't go into the loader
globalCodes["Translations"] = {}
globalCodes["Locales"] = {}
else:
if compileType == "build":
# also remove them here, as this info is now with the packages
globalCodes["Translations"] = {}
globalCodes["Locales"] = {}
plugCodeFile = compConf.get("code/decode-uris-plug", False)
if compileType == "build":
filepackages = [(x.file,) for x in packages]
bootContent = generateBootCode(parts, filepackages, boot, script, compConf, variants, settings, bootPackage, globalCodes, compileType, plugCodeFile, format)
else:
filepackages = [x.classes for x in packages]
bootContent = generateBootCode(parts, filepackages, boot, script, compConf, variants={}, settings={}, bootCode=None, globalCodes=globalCodes, version=compileType, decodeUrisFile=plugCodeFile, format=format)
return bootContent
def getPackageData(package):
data = {}
data["resources"] = package.data.resources
data["translations"] = package.data.translations
data["locales"] = package.data.locales
data = json.dumpsCode(data)
data += ';\n'
return data
def compilePackage(packageIndex, package):
self._console.info("Compiling package #%s:" % packageIndex, False)
self._console.indent()
# Compile file content
pkgCode = self._treeCompiler.compileClasses(package.classes, variants, optimize, format)
pkgData = getPackageData(package)
hash = sha.getHash(pkgData + pkgCode)[:12] # first 12 chars should be enough
isBootPackage = packageIndex == 0
if isBootPackage:
compiledContent = ("qx.$$packageData['%s']=" % hash) + pkgData + pkgCode
else:
compiledContent = u'''qx.$$packageData['%s']=%s\n''' % (hash, pkgData)
compiledContent += u'''qx.Part.$$notifyLoad("%s", function() {\n%s\n});''' % (hash, pkgCode)
#
package.hash = hash # to fill qx.$$loader.packageHashes in generateBootScript()
self._console.debug("Done: %s" % self._computeContentSize(compiledContent))
self._console.outdent()
return compiledContent
##
# takes an array of (po-data, locale-data) dict pairs
# merge all po data and all cldr data in a single dict each
def mergeTranslationMaps(transMaps):
poData = {}
cldrData = {}
for pac_dat, loc_dat in transMaps:
for loc in pac_dat:
if loc not in poData:
poData[loc] = {}
poData[loc].update(pac_dat[loc])
for loc in loc_dat:
if loc not in cldrData:
cldrData[loc] = {}
cldrData[loc].update(loc_dat[loc])
return (poData, cldrData)
# -- Main - runCompiled ------------------------------------------------
# Early return
compileType = self._job.get("compile/type", "")
if compileType not in ("build", "source"):
return
packages = script.packagesSortedSimple()
parts = script.parts
boot = script.boot
variants = script.variants
libraries = script.libraries
self._treeCompiler = treeCompiler
self._variants = variants
self._script = script
self._console.info("Generate %s version..." % compileType)
self._console.indent()
# - Evaluate job config ---------------------
# Compile config
compConf = self._job.get("compile-options")
compConf = ExtMap(compConf)
# Whether the code should be formatted
format = compConf.get("code/format", False)
script.scriptCompress = compConf.get("paths/gzip", False)
# Read in settings
settings = self.getSettings()
script.settings = settings
# Read libraries
libs = self._job.get("library", [])
# Get translation maps
locales = compConf.get("code/locales", [])
translationMaps = self.getTranslationMaps(packages, variants, locales)
# Read in base file name
fileRelPath = getOutputFile(compileType)
filePath = self._config.absPath(fileRelPath)
script.baseScriptPath = filePath
if compileType == "build":
# read in uri prefixes
scriptUri = compConf.get('uris/script', 'script')
scriptUri = Path.posifyPath(scriptUri)
fileUri = getFileUri(scriptUri)
# for resource list
resourceUri = compConf.get('uris/resource', 'resource')
resourceUri = Path.posifyPath(resourceUri)
else:
# source version needs place where the app HTML ("index.html") lives
self.approot = self._config.absPath(compConf.get("paths/app-root", ""))
resourceUri = None
scriptUri = None
# Get global script data (like qxlibraries, qxresources,...)
globalCodes = {}
globalCodes["Settings"] = settings
globalCodes["Variants"] = self.generateVariantsCode(variants)
globalCodes["Libinfo"] = self.generateLibInfoCode(libs, format, resourceUri, scriptUri)
# add synthetic output lib
if scriptUri: out_sourceUri= scriptUri
else:
out_sourceUri = self._computeResourceUri({'class': ".", 'path': os.path.dirname(script.baseScriptPath)}, OsPath(""), rType="class", appRoot=self.approot)
out_sourceUri = os.path.normpath(out_sourceUri.encodedValue())
globalCodes["Libinfo"]['__out__'] = { 'sourceUri': out_sourceUri }
globalCodes["Resources"] = self.generateResourceInfoCode(script, settings, libraries, format)
globalCodes["Translations"],\
globalCodes["Locales"] = mergeTranslationMaps(translationMaps)
# Potentally create dedicated I18N packages
i18n_as_parts = not self._job.get("packages/i18n-with-boot", True)
if i18n_as_parts:
script = self.generateI18NParts(script, globalCodes)
self.writePackages([p for p in script.packages if getattr(p, "__localeflag", False)], script)
if compileType == "build":
# - Specific job config ---------------------
# read in compiler options
optimize = compConf.get("code/optimize", [])
self._treeCompiler.setOptimize(optimize)
# - Generating packages ---------------------
self._console.info("Generating packages...")
self._console.indent()
bootPackage = ""
for packageIndex, package in enumerate(packages):
package.compiled = compilePackage(packageIndex, package)
self._console.outdent()
if not len(packages):
raise RuntimeError("No valid boot package generated.")
# - Put loader and packages together -------
loader_with_boot = self._job.get("packages/loader-with-boot", True)
# handle loader and boot package
if not loader_with_boot:
loadPackage = Package(0) # make a dummy Package for the loader
packages.insert(0, loadPackage)
# attach file names (do this before calling generateBootScript)
for package, fileName in zip(packages, self.packagesFileNames(script.baseScriptPath, len(packages))):
package.file = os.path.basename(fileName)
if self._job.get("compile-options/paths/scripts-add-hash", False):
package.file = self._fileNameWithHash(package.file, package.hash)
# generate and integrate boot code
if loader_with_boot:
# merge loader code with first package
bootCode = generateBootScript(globalCodes, script, packages[0].compiled)
packages[0].compiled = bootCode
else:
loaderCode = generateBootScript(globalCodes, script)
packages[0].compiled = loaderCode
# write packages
self.writePackages(packages, script)
# ---- 'source' version ------------------------------------------------
else:
sourceContent = generateBootScript(globalCodes, script, bootPackage="", compileType=compileType)
# Construct file name
resolvedFilePath = self._resolveFileName(filePath, variants, settings)
# Save result file
filetool.save(resolvedFilePath, sourceContent)
if compConf.get("paths/gzip"):
filetool.gzip(resolvedFilePath, sourceContent)
self._console.outdent()
self._console.debug("Done: %s" % self._computeContentSize(sourceContent))
self._console.outdent()
self._console.outdent()
return # runCompiled()
def from_doc_root_to_app_root(jobconf, confObj, doc_root):
japp_root = jobconf.get("compile-options/paths/app-root", "source")
app_root = os.path.normpath(os.path.join(confObj.absPath(japp_root), 'index.html'))
_, _, url_path = Path.getCommonPrefix(doc_root, app_root)
url_path = Path.posifyPath(url_path)
return url_path
def extractAssetPart(libresuri, imguri):
pre,libsfx,imgsfx = Path.getCommonPrefix(libresuri, imguri) # split libresuri from imguri
if imgsfx[0] == os.sep: imgsfx = imgsfx[1:] # strip leading '/'
return imgsfx # use the bare img suffix as its asset Id
def file2package(fname, rootname):
fileid = Path.getCommonPrefix(rootname, fname)[2]
fileid = os.path.splitext(fileid)[0] # strip .py
return fileid.replace(os.sep, '.')
def getFileUri(scriptUri):
appfile = os.path.basename(fileRelPath)
fileUri = os.path.join(scriptUri, appfile) # make complete with file name
fileUri = Path.posifyPath(fileUri)
return fileUri
ax1.plot(log_dict['gen'], log_dict['min'],
color='g', label="Minimum Fitness")
ax1.plot(log_dict['gen'], log_dict['max'],
color='b', label="Maximum Fitness")
ax1.legend()
save_file_path = get_save_file_path(log_dir_path, 'graph.png')
plt.savefig(str(save_file_path))
if __name__ == "__main__":
args = _get_args()
optimize_dir_path = get_save_dir_path(
root_optimize_dir_path, args.user_name, f'{args.image_name}/{args.image_number}')
image_dir = root_image_dir_path/args.image_name/args.image_number
image_path = list(image_dir.glob('*.*'))[0]
enhancer = ResizableEnhancer(str(image_path), IMAGE_SIZE)
model = RankNet(IMAGE_SHAPE)
model.load(args.weights_file_path)
image_generator = EnhanceGenerator(enhancer)
best_param_list, _ = Optimizer(model, image_generator, EnhanceDecorder()).optimize(20)
for index, best_param in enumerate(best_param_list):
save_path = str(Path(optimize_dir_path)/f'best_{index}.png')
enhancer.enhance(best_param).save(save_path)
def flatten(self):
a = [self.width, self.height, self.type, self.lib]
if self.mappedId:
a.extend([Path.posifyPath(self.mappedId), self.left, self.top])
return a
def flatten(self):
a = [self.width, self.height, self.type, self.lib]
if self.mappedId:
a.extend([Path.posifyPath(self.mappedId), self.left, self.top])
return a
def replaceWithNamespace(imguri, liburi, libns):
pre,libsfx,imgsfx = Path.getCommonPrefix(liburi, imguri)
if imgsfx[0] == os.sep: imgsfx = imgsfx[1:] # strip leading '/'
imgshorturi = os.path.join("${%s}" % libns, imgsfx)
return imgshorturi
def file2package(fname, rootname):
fileid = Path.getCommonPrefix(rootname, fname)[2]
fileid = os.path.splitext(fileid)[0] # strip .py
return fileid.replace(os.sep, '.')
def findAllResources(self, libraries, filter=None):
"""Find relevant resources/assets, implementing shaddowing of resources.
Returns a list of resources, each a pair of [file_path, uri]"""
# - Helpers -----------------------------------------------------------
def getCache(lib):
cacheId = "resinlib-%s" % lib._path
liblist = self._genobj._cache.read(cacheId, dependsOn=None, memory=True)
return liblist, cacheId
def isSkipFile(f):
if [x for x in map(lambda x: re.search(x, f), ignoredFiles) if x!=None]:
return True
else:
return False
def resourceValue(r):
# create a pair res = [path, uri] for this resource...
rsource = os.path.normpath(r) # normalize "./..."
relpath = (Path.getCommonPrefix(libObj._resourcePath, rsource))[2]
if relpath[0] == os.sep: # normalize "/..."
relpath = relpath[1:]
ruri = (self._genobj._computeResourceUri(lib, relpath, rType='resource',
appRoot=self._genobj.approot))
return (rsource, ruri)
# - Main --------------------------------------------------------------
result = []
cacheList = [] # to poss. populate cache
cacheId = "" # will be filled in getCache()
ignoredFiles = [r'\.meta$',] # files not considered as resources
libs = libraries[:]
#libs.reverse() # this is to search the 'important' libs first
# go through all libs (weighted) and collect necessary resources
for lib in libs:
# create wrapper object
libObj = LibraryPath(lib, self._genobj._console)
# retrieve list of library resources
libList, cacheId = getCache(libObj)
if libList:
inCache = True
else:
libList = libObj.scanResourcePath()
inCache = False
# go through list of library resources and add suitable
for resource in libList:
# scanResourcePath() yields absolute paths to a resource, but
# we only want to match against the 'resource' part of it
resourcePart = Path.getCommonPrefix(libObj._resourcePath, resource)[2]
if not inCache:
cacheList.append(resource)
if isSkipFile(resource):
continue
elif (filter and not filter(resourcePart)):
continue
else:
result.append(resource)
if not inCache:
# cache write
self._genobj._cache.write(cacheId, cacheList, memory=True, writeToFile=False)
return result
