def generateResourceInfoCode(self, script, settings, libraries, format=False):
def addResourceInfoToPackages(script):
for package in script.packages:
package_resources = []
# TODO: the next is a hack, since package.classes are still id's
package_classes = [x for x in script.classesObj if x.id in package.classes]
for clazz in package_classes:
package_resources.extend(clazz.resources)
package.data.resources = Script.createResourceStruct(package_resources, formatAsTree=resources_tree,
updateOnlyExistingSprites=True)
return
# -- main --------------------------------------------------------------
compConf = self._job.get ("compile-options")
compConf = ExtMap (compConf)
resources_tree = compConf.get ("code/resources-tree", False)
classes = Class.mapResourcesToClasses (libraries, script.classesObj,
self._job.get("asset-let", {}))
filteredResources = []
for clazz in classes:
filteredResources.extend(clazz.resources)
resdata = Script.createResourceStruct (filteredResources, formatAsTree=resources_tree,
updateOnlyExistingSprites=True)
# add resource info to packages
addResourceInfoToPackages(script)
return resdata # end: generateResourceInfoCode()
def runResources(confObj, script):
jobconf = script.jobconfig
if not jobconf.get("copy-resources", False):
return
console = Context.console
console.info("Copying resources...")
classList = script.classesObj
resTargetRoot = jobconf.get("copy-resources/target", "build")
resTargetRoot = confObj.absPath(resTargetRoot)
#generator.approot = resTargetRoot # doesn't seem necessary anymore
# map resources to class.resources
classList = Class.mapResourcesToClasses(script.libraries, classList,
jobconf.get("asset-let", {}))
console.indent()
# make resources to copy unique
resources_to_copy = set(_res for cls in classList
for _res in cls.resources)
# Copy resources
#for lib in libs:
for res in resources_to_copy:
# construct target path
resTarget = os.path.join(resTargetRoot, 'resource', res.id)
# Copy
_copyResources(res.path, os.path.dirname(resTarget))
console.outdent()
def runResources(confObj, script):
jobconf = script.jobconfig
if not jobconf.get("copy-resources", False):
return
console = Context.console
console.info("Copying resources...")
classList = script.classesObj
resTargetRoot = jobconf.get("copy-resources/target", "build")
resTargetRoot = confObj.absPath(resTargetRoot)
# generator.approot = resTargetRoot # doesn't seem necessary anymore
# map resources to class.resources
classList = Class.mapResourcesToClasses(script.libraries, classList, jobconf.get("asset-let", {}))
console.indent()
# make resources to copy unique
resources_to_copy = set(_res for cls in classList for _res in cls.resources)
# Copy resources
# for lib in libs:
for res in resources_to_copy:
# construct target path
resTarget = os.path.join(resTargetRoot, "resource", res.id)
# Copy
_copyResources(res.path, os.path.dirname(resTarget))
console.outdent()
def makeClassObj(self, filePathId, filePath, props):
##
# provide a default context dict
def get_contextdict():
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
return contextdict
# -----------------------------------------------------------------------
contextdict = get_contextdict(
) # TODO: Clazz() still relies on a context dict!
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict)
else:
clazz = Class(filePathId, filePath, self, contextdict)
# extract code ID (e.g. class name, mixin name, ...)
#fileCodeId = self.checkClassId(clazz, filePathId) # involves parsing
clazz.encoding = props.fileEncoding
clazz.size = props.fileSize # dependency logging uses this
clazz.package = props.filePackage # Apiloader uses this
clazz.relpath = props.fileRel # Locale uses this
clazz.m_time_ = props.fileStat.st_mtime
return clazz, filePathId
def packagesResourceInfo(self, script):
classes = Class.mapResourcesToClasses (script.libraries, script.classesObj,
self._job.get("asset-let", {}))
for package in script.packages:
package_resources = []
package_classes = package.classes
for clazz in package_classes:
package_resources.extend(clazz.resources)
package.data.resources = Script.createResourceStruct(package_resources, formatAsTree=False,
updateOnlyExistingSprites=True)
return script
def makeClassObj(self, filePathId, filePath, props):
##
# provide a default context dict
def get_contextdict():
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
return contextdict
# -----------------------------------------------------------------------
contextdict = get_contextdict() # TODO: Clazz() still relies on a context dict!
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict)
else:
clazz = Class(filePathId, filePath, self, contextdict)
# extract code ID (e.g. class name, mixin name, ...)
#fileCodeId = self.checkClassId(clazz, filePathId) # involves parsing
clazz.encoding = props.fileEncoding
clazz.size = props.fileSize # dependency logging uses this
clazz.package = props.filePackage # Apiloader uses this
clazz.relpath = props.fileRel # Locale uses this
clazz.m_time_ = props.fileStat.st_mtime
return clazz, filePathId
def _checkCache(self, fileId, variants, optimize, format=False):
filePath = self._classes[fileId].path
classVariants = self._classes[fileId].classVariants()
relevantVariants = Class.projectClassVariantsToCurrent(classVariants, variants)
variantsId = util.toString(relevantVariants)
optimizeId = self.generateOptimizeId(optimize)
cacheId = "compiled-%s-%s-%s-%s" % (filePath, variantsId, optimizeId, format)
compiled, _ = self._cache.read(cacheId, filePath)
return cacheId, compiled
def checkCache(self, fileId, variants, optimize, format=False):
filePath = self._classes[fileId]["path"]
classVariants = self._classesObj[fileId].classVariants()
relevantVariants = Class.projectClassVariantsToCurrent(classVariants, variants)
variantsId = util.toString(relevantVariants)
optimizeId = self.generateOptimizeId(optimize)
cacheId = "compiled-%s-%s-%s-%s" % (filePath, variantsId, optimizeId, format)
compiled, _ = self._cache.read(cacheId, filePath)
return cacheId, compiled
def dependencies(self, classId, classPath, variantSet, force=False):
print "\nrunning on Pyro server", str(self.num)
c = Class(classId, classPath, None, generator_context, self.classList)
d, cached = c.dependencies(variantSet, force)
return d, cached
def _scanClassPath(self, timeOfLastScan=0):
codeIdFromTree = True # switch between regex- and tree-based codeId search
# Check class path
classPath = os.path.join(self.path, self.classPath)
if not os.path.isdir(classPath):
raise ConfigurationError("Class path from Manifest doesn't exist: %s" % self.classPath)
# Check multiple namespaces
if not len([d for d in os.listdir(classPath) if not d.startswith(".")]) == 1:
self._console.warn(
"The class path must contain exactly one namespace; ignoring everything else: '%s'" % (classPath,)
)
# Check Manifest namespace matches file system
nsPrefix = self.namespace.replace(".", os.sep)
classNSRoot = os.path.join(classPath, nsPrefix)
if not os.path.isdir(classNSRoot):
raise ValueError("Manifest namespace does not exist on file system: '%s'" % (classNSRoot))
self._console.debug("Scanning class folder...")
classList = []
existClassIds = dict([(x.id, x) for x in self._classes]) # if we scanned before
docs = {}
# TODO: Clazz still relies on a context dict!
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
# Iterate...
for root, dirs, files in filetool.walk(classNSRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Add good directories
currNameSpace = root[len(classNSRoot + os.sep) :]
currNameSpace = currNameSpace.replace(os.sep, ".")
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(fileName):
continue
self._console.dot()
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(classNSRoot + os.sep, "")
fileExt = os.path.splitext(fileName)[-1]
fileStat = os.stat(filePath)
fileSize = fileStat.st_size
fileMTime = fileStat.st_mtime
# Compute full URI from relative path
fileUri = self.classUri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
filePathId = self.namespace + "." + filePathId
filePathId = unidata.normalize("NFC", filePathId) # combine combining chars: o" -> ö
# check if known and fresh
if filePathId in existClassIds and fileMTime < timeOfLastScan:
classList.append(existClassIds[filePathId])
# print "re-using existing", filePathId
continue # re-use known class
# Extract package ID
filePackage = filePathId[: filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[: filePathId.rfind(".")]
docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": self.encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize,
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict, self._classesObj)
else:
clazz = Class(filePathId, filePath, self, contextdict, self._classesObj)
# Extract code ID (e.g. class name, mixin name, ...)
try:
if codeIdFromTree:
fileCodeId = self._getCodeId(clazz)
else:
# Read content
fileContent = filetool.read(filePath, self.encoding)
fileCodeId = self._getCodeId1(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u" (%s)" % fileName
e.args = tuple(argsList)
raise e
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error("Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("Path: %s" % fileRel)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
clazz.encoding = self.encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
clazz.relpath = fileRel # Locale uses this
clazz.m_time_ = fileStat.st_mtime
classList.append(clazz)
def _scanClassPath(self, timeOfLastScan=0):
##
# provide a default context dict
def get_contextdict():
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
return contextdict
##
# check class path is on file system
def check_class_path(classRoot):
if not os.path.isdir(classRoot):
raise ConfigurationError("Class path from Manifest doesn't exist: %s" % self.classPath)
##
# check single subdirectory from class path
def check_multiple_namespaces(classRoot):
if not len([d for d in os.listdir(classRoot) if not d.startswith(".")]) == 1:
self._console.warn("The class path should contain exactly one namespace: '%s'" % (classRoot,))
##
# check manifest namespace is on file system
def check_manifest_namespace(classRoot):
nsPrefix = self.namespace.replace(".", os.sep)
classNSRoot = os.path.join(classRoot, nsPrefix)
if not os.path.isdir(classNSRoot):
raise ValueError("Manifest namespace does not exist on file system: '%s'" % (classNSRoot))
# ----------------------------------------------------------------------
codeIdFromTree = True # switch between regex- and tree-based codeId search
classList = []
existClassIds = dict([(x.id, x) for x in self._classes]) # if we scanned before
docs = {}
contextdict = get_contextdict() # TODO: Clazz() still relies on a context dict!
classRoot = os.path.join(self.path, self.classPath)
check_class_path(classRoot)
check_multiple_namespaces(classRoot)
check_manifest_namespace(classRoot)
self._console.debug("Scanning class folder...")
# Iterate...
for root, dirs, files in filetool.walk(classRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(fileName):
continue
self._console.dot()
# basic attributes
filePath = os.path.join(root, fileName) # /foo/bar/baz/source/class/my/space/AppClass.js
fileRel = filePath.replace(classRoot + os.sep, "") # my/space/AppClass.js
fileExt = os.path.splitext(fileName)[-1] # "js"
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".") # my.space.AppClass
filePathId = unidata.normalize("NFC", filePathId) # o" -> ö
filePackage = filePathId[: filePathId.rfind(".")] if "." in filePathId else "" # my.space
fileStat = os.stat(filePath)
fileSize = fileStat.st_size
fileMTime = fileStat.st_mtime
# ignore non-script
if fileExt != ".js":
continue
# check if known and fresh
if filePathId in existClassIds and fileMTime < timeOfLastScan:
classList.append(existClassIds[filePathId])
continue # re-use known class
# handle doc files
if fileName == self._docFilename:
docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": self.encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize,
}
# stop further processing
continue
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict)
else:
clazz = Class(filePathId, filePath, self, contextdict)
# extract code ID (e.g. class name, mixin name, ...)
try:
if codeIdFromTree:
fileCodeId = self._getCodeId(clazz)
else:
# use regexp
fileContent = filetool.read(filePath, self.encoding)
fileCodeId = self._getCodeId1(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u" (%s)" % fileName
e.args = tuple(argsList)
raise e
# ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# compare path and content
if fileCodeId != filePathId:
errmsg = [
u"Detected conflict between filename and classname!\n",
u" Classname: %s\n" % fileCodeId,
u" Path: %s\n" % filePath,
]
raise RuntimeError(u"".join(errmsg))
# store file data
self._console.debug("Adding class %s" % filePathId)
clazz.encoding = self.encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
clazz.relpath = fileRel # Locale uses this
clazz.m_time_ = fileStat.st_mtime
classList.append(clazz)
def optimizeEnvironmentClass(envClass, compConf):
assert envClass.id == "qx.core.Environment"
tree = Class.optimizeEnvironmentClass(envClass, compConf)
code = Packer().serializeNode(tree, None, [u''], compConf.format)
code = u''.join(code)
return code
def depsToProviderFormat(classDepsIter, depsLogConf):
##
# duplicates CodeProvider.passesOutputFilter
def passesOutputFilter(resId):
# must match some include expressions
if not filter(None, [x.search(resId) for x in inclregexps]): # [None, None, _sre.match, None, _sre.match, ...]
return False
# must not match any exclude expressions
if filter(None, [x.search(resId) for x in exclregexps]):
return False
return True
# ---------------------------------------
inclregexps = jobconf.get("provider/include", ["*"])
exclregexps = jobconf.get("provider/exclude", [])
inclregexps = map(textutil.toRegExp, inclregexps)
exclregexps = map(textutil.toRegExp, exclregexps)
replace_dots = depsLogConf.get("json/replace-dots-in", [])
slashes_keys = 'keys' in replace_dots
slashes_vals = 'values' in replace_dots
classToDeps = {}
# Class deps
for (packageId, classId, depId, loadOrRun) in classDepsIter:
if passesOutputFilter(classId):
if classId not in classToDeps:
classToDeps[classId] = {}
classToDeps[classId]["load"] = []
classToDeps[classId]["run"] = []
if depId != None:
classToDeps[classId][loadOrRun].append(depId)
if slashes_vals:
# transform dep items
for key, val in classToDeps.items():
newval = []
for ldep in val["load"]:
newdep = ldep.replace(".", "/")
newval.append(newdep)
val["load"] = newval
newval = []
for ldep in val["run"]:
newdep = ldep.replace(".", "/")
newval.append(newdep)
val["run"] = newval
# Resource deps
# class list
classObjs = [x for x in script.classesObj if x.id in classToDeps.keys()]
# map resources to class.resources
classObjs = Class.mapResourcesToClasses(script.libraries, classObjs, jobconf.get("asset-let", {}))
for clazz in classObjs:
reskeys = ["/resource/resources#"+x.id for x in clazz.resources]
classToDeps[clazz.id]["run"].extend(reskeys)
# Message key deps
for classId in classToDeps:
#classKeys, _ = Locale.getTranslation(classId, {})
classKeys, _ = script.classesAll[classId].messageStrings({})
transIds = set(x['id'] for x in classKeys) # get the msgid's, uniquely
transIds.update(x['plural'] for x in classKeys if 'plural' in x) # add plural keys
transKeys = ["/translation/i18n-${lang}#" + x for x in transIds]
classToDeps[classId]["run"].extend(transKeys)
# CLDR dependency
for classId in classToDeps:
if script.classesAll[classId].getHints("cldr"):
classToDeps[classId]["run"].append("/locale/locale-${lang}#cldr")
if slashes_keys:
# transform dep keys ("qx.Class" -> "qx/Class.js")
for key, val in classToDeps.items():
if key.find(".")>-1:
newkey = key.replace(".", "/")
classToDeps[newkey] = classToDeps[key]
del classToDeps[key]
# sort information for each class (for stable output)
for classvals in classToDeps.values():
for key in classvals:
classvals[key] = sorted(classvals[key], reverse=True)
# write to file
file_ = depsLogConf.get('json/file', "deps.json")
console.info("Writing dependency data to file: %s" % file_)
pretty = depsLogConf.get('json/pretty', None)
if pretty:
indent = 2
separators = (', ', ': ')
else:
indent = None
separators = (',', ':')
filetool.save(file_, json.dumps(classToDeps, sort_keys=True, indent=indent, separators=separators))
return
def depsToProviderFormat(classDepsIter, depsLogConf):
##
# duplicates CodeProvider.passesOutputFilter
def passesOutputFilter(resId):
# must match some include expressions
if not filter(None, [
x.search(resId) for x in inclregexps
]): # [None, None, _sre.match, None, _sre.match, ...]
return False
# must not match any exclude expressions
if filter(None, [x.search(resId) for x in exclregexps]):
return False
return True
# ---------------------------------------
inclregexps = jobconf.get("provider/include", ["*"])
exclregexps = jobconf.get("provider/exclude", [])
inclregexps = map(textutil.toRegExp, inclregexps)
exclregexps = map(textutil.toRegExp, exclregexps)
replace_dots = depsLogConf.get("json/replace-dots-in", [])
slashes_keys = 'keys' in replace_dots
slashes_vals = 'values' in replace_dots
classToDeps = {}
# Class deps
for (packageId, classId, depId, loadOrRun) in classDepsIter:
if passesOutputFilter(classId):
if classId not in classToDeps:
classToDeps[classId] = {}
classToDeps[classId]["load"] = []
classToDeps[classId]["run"] = []
if depId != None:
classToDeps[classId][loadOrRun].append(depId)
if slashes_vals:
# transform dep items
for key, val in classToDeps.items():
newval = []
for ldep in val["load"]:
newdep = ldep.replace(".", "/")
newval.append(newdep)
val["load"] = newval
newval = []
for ldep in val["run"]:
newdep = ldep.replace(".", "/")
newval.append(newdep)
val["run"] = newval
# Resource deps
# class list
classObjs = [
x for x in script.classesObj if x.id in classToDeps.keys()
]
# map resources to class.resources
classObjs = Class.mapResourcesToClasses(script.libraries, classObjs,
jobconf.get("asset-let", {}))
for clazz in classObjs:
reskeys = ["/resource/resources#" + x.id for x in clazz.resources]
classToDeps[clazz.id]["run"].extend(reskeys)
# Message key deps
for classId in classToDeps:
#classKeys, _ = Locale.getTranslation(classId, {})
classKeys, _ = script.classesAll[classId].messageStrings({})
transIds = set(x['id']
for x in classKeys) # get the msgid's, uniquely
transIds.update(x['plural'] for x in classKeys
if 'plural' in x) # add plural keys
transKeys = ["/translation/i18n-${lang}#" + x for x in transIds]
classToDeps[classId]["run"].extend(transKeys)
# CLDR dependency
for classId in classToDeps:
if script.classesAll[classId].getHints("cldr"):
classToDeps[classId]["run"].append(
"/locale/locale-${lang}#cldr")
if slashes_keys:
# transform dep keys ("qx.Class" -> "qx/Class.js")
for key, val in classToDeps.items():
if key.find(".") > -1:
newkey = key.replace(".", "/")
classToDeps[newkey] = classToDeps[key]
del classToDeps[key]
# sort information for each class (for stable output)
for classvals in classToDeps.values():
for key in classvals:
classvals[key] = sorted(classvals[key], reverse=True)
# write to file
file_ = depsLogConf.get('json/file', "deps.json")
console.info("Writing dependency data to file: %s" % file_)
pretty = depsLogConf.get('json/pretty', None)
if pretty:
indent = 2
separators = (', ', ': ')
else:
indent = None
separators = (',', ':')
filetool.save(
file_,
json.dumps(classToDeps,
sort_keys=True,
indent=indent,
separators=separators))
return
def _scanClassPath(self, timeOfLastScan=0):
codeIdFromTree = True # switch between regex- and tree-based codeId search
# Check class path
classPath = os.path.join(self.path, self.classPath)
if not os.path.isdir(classPath):
raise ConfigurationError(
"Class path from Manifest doesn't exist: %s" % self.classPath)
# Check multiple namespaces
if not len([d for d in os.listdir(classPath) if not d.startswith(".")
]) == 1:
self._console.warn(
"The class path must contain exactly one namespace; ignoring everything else: '%s'"
% (classPath, ))
# Check Manifest namespace matches file system
nsPrefix = self.namespace.replace(".", os.sep)
classNSRoot = os.path.join(classPath, nsPrefix)
if not os.path.isdir(classNSRoot):
raise ValueError(
"Manifest namespace does not exist on file system: '%s'" %
(classNSRoot))
self._console.debug("Scanning class folder...")
classList = []
existClassIds = dict([(x.id, x)
for x in self._classes]) # if we scanned before
docs = {}
# TODO: Clazz still relies on a context dict!
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
# Iterate...
for root, dirs, files in filetool.walk(classNSRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Add good directories
currNameSpace = root[len(classNSRoot + os.sep):]
currNameSpace = currNameSpace.replace(os.sep,
".") # TODO: var name
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(
fileName):
continue
self._console.dot()
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(
classNSRoot + os.sep,
"") # now only path fragment *afte* NS
fileExt = os.path.splitext(fileName)[-1]
fileStat = os.stat(filePath)
fileSize = fileStat.st_size
fileMTime = fileStat.st_mtime
# Compute full URI from relative path
fileUri = self.classUri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
filePathId = self.namespace + "." + filePathId # e.g. "qx.core.Environment"
filePathId = unidata.normalize(
"NFC", filePathId) # combine combining chars: o" -> ö
fileId = nsPrefix + "/" + fileRel # e.g. "qx/core/Environment.js"
# check if known and fresh
if (filePathId in existClassIds
and fileMTime < timeOfLastScan):
classList.append(existClassIds[filePathId])
#print "re-using existing", filePathId
continue # re-use known class
# Extract package ID
filePackage = filePathId[:filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[:filePathId.rfind(".")]
docs[filePackage] = {
"relpath": fileId,
"path": filePath,
"encoding": self.encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict)
else:
clazz = Class(filePathId, filePath, self, contextdict)
# Extract code ID (e.g. class name, mixin name, ...)
try:
if codeIdFromTree:
fileCodeId = self._getCodeId(clazz)
else:
# Read content
fileContent = filetool.read(filePath, self.encoding)
fileCodeId = self._getCodeId1(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u' (%s)' % fileName
e.args = tuple(argsList)
raise e
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error(
"Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("Path: %s" % filePath)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
clazz.encoding = self.encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
clazz.relpath = fileId # Locale uses this
clazz.m_time_ = fileStat.st_mtime
classList.append(clazz)
# calculate dependencies and add required classes
classlistFromClassRecursive(depsItem, excludeWithDeps, variants, result, warn_deps, [], allowBlockLoaddeps)
#classlistFromClassIterative(depsItem, excludeWithDeps, variants, result, warn_deps, [], allowBlockLoaddeps)
self._console.dotclear()
#print len(result)," ",
# process qx.core.Environment
if ("qx.core.Environment" in resultNames
and "variants" in script.optimize
and not processedEnvironment):
envObj = self._classesObj["qx.core.Environment"]
envTreeId = "tree-%s-%s" % (envObj.path, util.toString({})) # TODO: {} is a temp. hack
compOpts = CompileOptions(optimize=[], variants=variants)
compOpts.allClassVariants = script.classVariants([self._classesObj[x] for x in resultNames])
tree = Class.optimizeEnvironmentClass(envObj, compOpts)
self._cache.write(envTreeId, tree, memory=True, writeToFile=False)
# this is for the side-effect of leaving a modified tree for qx.core.Environmet
# in the cache!
_ = envObj.dependencies(variants, force=True)
# this is for the side-effect of re-calculating the transitive dependencies
# of qx.core.Environment!
processedEnvironment = True
else:
# We currently know that one additional iteration is enough,
# after optimizeEnvironmentClass() has run. This has to do
# with the fact that it only removes dependencies to
# qx.bom.client.* classes, which in turn do not use
# qx.core.Env calls. So after calculating the new class
# list, allClassVariants will not have changed. If it would,
# we would need to re-calculate until the class list is
def tree(self, classId, classPath, variantSet):
c = Class(classId, classPath, None, generator_context, {})
t = c.tree(variantSet)
return t
def _scanClassPath(self, path, uri, encoding):
if not os.path.exists(path):
raise ValueError("The given class path does not exist: %s" % path)
self._console.debug("Scanning class folder...")
classList = {}
docs = {}
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirectories.match(ignoredDir):
dirs.remove(ignoredDir)
# Add good directories
currNameSpace = root[len(path+os.sep):]
currNameSpace = currNameSpace.replace(os.sep, ".")
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith("."):
continue
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(path + os.sep, "")
fileExt = os.path.splitext(fileName)[-1]
fileSize = os.stat(filePath).st_size
# Compute full URI from relative path
fileUri = uri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
# Extract package ID
filePackage = filePathId[:filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[:filePathId.rfind(".")]
docs[filePackage] = {
"relpath" : fileRel,
"path" : filePath,
"encoding" : encoding,
"namespace" : self.namespace,
"id" : filePathId,
"package" : filePackage,
"size" : fileSize
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
# Read content
fileContent = filetool.read(filePath, encoding)
# Extract code ID (e.g. class name, mixin name, ...)
try:
fileCodeId = self._getCodeId(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u' (%s)' % fileName
e.args = tuple(argsList)
raise e
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error("Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("Path: %s" % fileRel)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
classList[filePathId] = {
"relpath" : fileRel,
"path" : filePath,
"encoding" : encoding,
"namespace" : self.namespace,
"id" : filePathId,
"package" : filePackage,
"size" : fileSize
}
# TODO: Clazz still relies on a context dict!
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
# TODO: currently creation of throw-away objects (unless they're .append'ed)
clazz = Class(classList[filePathId], filePath, self, contextdict, self._classesObj)
clazz.encoding = encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
def tree(self, classId, classPath, variantSet):
c = Class(classId, classPath, None, generator_context, {})
t = c.tree()
return t
def dependencies(self, classId, classPath, variantSet, force=False):
print "\nrunning on Pyro server", str(self.num)
c = Class(classId, classPath, None, generator_context, self.classList)
d, cached = c.dependencies(variantSet, force)
return d, cached
def _scanClassPath(self, path, uri, encoding):
if not os.path.exists(path):
raise ValueError("The given class path does not exist: %s" % path)
self._console.debug("Scanning class folder...")
classList = {}
docs = {}
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirectories.match(ignoredDir):
dirs.remove(ignoredDir)
# Add good directories
currNameSpace = root[len(path + os.sep):]
currNameSpace = currNameSpace.replace(os.sep, ".")
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith("."):
continue
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(path + os.sep, "")
fileExt = os.path.splitext(fileName)[-1]
fileSize = os.stat(filePath).st_size
# Compute full URI from relative path
fileUri = uri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
# Extract package ID
filePackage = filePathId[:filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[:filePathId.rfind(".")]
docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
# Read content
fileContent = filetool.read(filePath, encoding)
# Extract code ID (e.g. class name, mixin name, ...)
try:
fileCodeId = self._getCodeId(fileContent)
except ValueError, e:
e.args[0] = e.args[0] + u' (%s)' % fileName
raise e
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error(
"Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("filePathId: %s" % filePathId)
self._console.error("Path: %s" % fileRel)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
classList[filePathId] = {
"relpath": fileRel,
"path": filePath,
"encoding": encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# TODO: Clazz still relies on a context dict!
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
# TODO: currently creation of throw-away objects (unless they're .append'ed)
clazz = Class(classList[filePathId], filePath, self,
contextdict, self._classesObj)
clazz.encoding = encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
def _scanClassPath(self, timeOfLastScan=0):
##
# provide a default context dict
def get_contextdict():
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
return contextdict
##
# check class path is on file system
def check_class_path(classRoot):
if not os.path.isdir(classRoot):
raise ConfigurationError(
"Class path from Manifest doesn't exist: %s" %
self.classPath)
##
# check single subdirectory from class path
def check_multiple_namespaces(classRoot):
try:
self._checkNamespace(classRoot)
except ValueError:
self._console.warn(
"The class path should contain exactly one namespace: '%s'"
% (classRoot, ))
##
# check manifest namespace is on file system
def check_manifest_namespace(classRoot):
nsPrefix = self.namespace.replace(".", os.sep)
classNSRoot = os.path.join(classRoot, nsPrefix)
if not os.path.isdir(classNSRoot):
raise ValueError(
"Manifest namespace does not exist on file system: '%s'" %
(classNSRoot))
# ----------------------------------------------------------------------
codeIdFromTree = True # switch between regex- and tree-based codeId search
classList = []
existClassIds = dict([(x.id, x)
for x in self._classes]) # if we scanned before
docs = {}
contextdict = get_contextdict(
) # TODO: Clazz() still relies on a context dict!
classRoot = os.path.join(self.path, self.classPath)
check_class_path(classRoot)
check_multiple_namespaces(classRoot)
check_manifest_namespace(classRoot)
self._console.debug("Scanning class folder...")
# Iterate...
for root, dirs, files in filetool.walk(classRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(
fileName):
continue
self._console.dot()
# basic attributes
filePath = os.path.join(
root,
fileName) # /foo/bar/baz/source/class/my/space/AppClass.js
fileRel = filePath.replace(classRoot + os.sep,
'') # my/space/AppClass.js
fileExt = os.path.splitext(fileName)[-1] # "js"
filePathId = fileRel.replace(fileExt, "").replace(
os.sep, ".") # my.space.AppClass
filePathId = unidata.normalize("NFC", filePathId) # o" -> ö
filePackage = filePathId[:filePathId.rfind(
".")] if "." in filePathId else "" # my.space
fileStat = os.stat(filePath)
fileSize = fileStat.st_size
fileMTime = fileStat.st_mtime
# ignore non-script
if fileExt != ".js":
continue
# check if known and fresh
if (filePathId in existClassIds
and fileMTime < timeOfLastScan):
classList.append(existClassIds[filePathId])
continue # re-use known class
# handle doc files
if fileName == self._docFilename:
docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": self.encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# stop further processing
continue
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict)
else:
clazz = Class(filePathId, filePath, self, contextdict)
# extract code ID (e.g. class name, mixin name, ...)
try:
if codeIdFromTree:
fileCodeId = self._getCodeId(clazz)
else:
# use regexp
fileContent = filetool.read(filePath, self.encoding)
fileCodeId = self._getCodeId1(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u' (%s)' % fileName
e.args = tuple(argsList)
raise e
# ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# compare path and content
if fileCodeId != filePathId:
errmsg = [
u"Detected conflict between filename and classname!\n",
u" Classname: %s\n" % fileCodeId,
u" Path: %s\n" % filePath,
]
raise RuntimeError(u''.join(errmsg))
# store file data
self._console.debug("Adding class %s" % filePathId)
clazz.encoding = self.encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
clazz.relpath = fileRel # Locale uses this
clazz.m_time_ = fileStat.st_mtime
classList.append(clazz)