def testTwoLinksSameTarget(self):
"""
dir1
link1 -> ../dir2
link2 -> ../dir2
dir2
foo.txt
Should follow the symlinks and find foo.txt twice
"""
self.tempDirs.append(tempfile.mkdtemp())
symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w")
symTarget1 = "link1"
symTarget1Path = os.path.join(self.tempDirs[0], symTarget1)
os.symlink(self.tempDirs[1], symTarget1Path)
symTarget2 = "link2"
symTarget2Path = os.path.join(self.tempDirs[0], symTarget2)
os.symlink(self.tempDirs[1], symTarget2Path)
expectedFiles = [ os.path.join(self.tempDirs[0], symTarget2, "foo.txt"),
os.path.join(self.tempDirs[0], symTarget1, "foo.txt") ]
foundFiles = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
foundFiles.append(os.path.join(root, filename))
self.failUnlessEqual(foundFiles, expectedFiles)
def scanRepository(self):
console.info("Scanning %s" % self.dir)
console.indent()
demoDir = "%sdemo%s" % (os.sep, os.sep)
manifestPaths = []
for root, dirs, files in filetool.walk(self.dir, topdown=True):
for dir in dirs:
if dir == "qooxdoo" or dir == "demo":
dirs.remove(dir)
continue
path = os.path.join(root, dir)
manifestPath = os.path.join(path, "Manifest.json")
if demoDir in manifestPath:
dirs.remove(dir)
continue
if os.path.isfile(manifestPath):
console.debug("Found manifest: " + repr(manifestPath))
manifestPaths.append(manifestPath)
console.info("Found %s manifests" % len(manifestPaths))
console.outdent()
return manifestPaths
def scanRepository(self):
console.info("Scanning %s" % self.dir)
console.indent()
demoDir = "%sdemo%s" % (os.sep, os.sep)
manifestPaths = []
for root, dirs, files in filetool.walk(self.dir, topdown=True):
for dir in dirs:
if dir == "qooxdoo" or dir == "demo":
dirs.remove(dir)
continue
path = os.path.join(root, dir)
manifestPath = os.path.join(path, "Manifest.json")
if demoDir in manifestPath:
dirs.remove(dir)
continue
if os.path.isfile(manifestPath):
console.debug("Found manifest: " + repr(manifestPath))
manifestPaths.append(manifestPath)
console.info("Found %s manifests" % len(manifestPaths))
console.outdent()
return manifestPaths
def _scanTranslationPath(self, path):
if not os.path.exists(path):
self._console.warn("The given path does not contain a translation folder: %s" % path)
self._console.debug("Scanning translation folder...")
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# Ignore non-script and dot files
if os.path.splitext(fileName)[-1] != ".po" or fileName.startswith("."):
continue
filePath = os.path.join(root, fileName)
fileLocale = os.path.splitext(fileName)[0]
self._translations[fileLocale] = self.translationEntry(fileLocale, filePath, self.namespace)
self._console.indent()
self._console.debug("Found %s translations" % len(self._translations))
self._console.outdent()
def _scanTranslationPath(self):
translations = {} # reset
if self.assets['translations']['path'] is None or not os.path.isdir(
os.path.join(self.path,self.assets['translations']['path'])):
self._console.info("Lib<%s>: Skipping non-existing translation path" % self.namespace)
return translations
path = os.path.join(self.path,self.assets['translations']['path'])
self._console.debug("Scanning translation folder...")
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# Ignore non-po and dot files
if os.path.splitext(fileName)[-1] != ".po" or fileName.startswith("."):
continue
filePath = os.path.join(root, fileName)
fileLocale = os.path.splitext(fileName)[0]
translations[fileLocale] = self.translationEntry(fileLocale, filePath, self.namespace)
self._console.indent()
self._console.debug("Found %s translations" % len(translations))
self._console.outdent()
return translations
def testSymlinkCyclic(self):
"""
dir
link -> ../dir
foo.txt
Should find the directories link and link/link, but not link/link/link
Should find the files foo.txt and link/foo.txt, but not link/link/foo.txt
"""
symTarget = "link"
fileInSource = file(os.path.join(self.tempDirs[0], "foo.txt"), "w")
fileInSource.close()
symTargetPath = os.path.join(self.tempDirs[0], symTarget)
os.symlink(self.tempDirs[0], symTargetPath)
expectedDirs = [os.path.join(symTargetPath, symTarget), symTargetPath]
expectedFiles = [
os.path.join(self.tempDirs[0], symTarget, "foo.txt"),
os.path.join(self.tempDirs[0], "foo.txt")
]
foundDirs = []
foundFiles = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for dirname in dirs:
foundDirs.append(os.path.join(root, dirname))
for filename in files:
foundFiles.append(os.path.join(root, filename))
self.failUnlessEqual(foundDirs, expectedDirs)
self.failUnlessEqual(foundFiles, expectedFiles)
def testSymlinkCyclic(self):
"""
dir
link -> ../dir
foo.txt
Should find the directories link and link/link, but not link/link/link
Should find the files foo.txt and link/foo.txt, but not link/link/foo.txt
"""
symTarget = "link"
fileInSource = file(os.path.join(self.tempDirs[0], "foo.txt"), "w")
fileInSource.close()
symTargetPath = os.path.join(self.tempDirs[0], symTarget)
os.symlink(self.tempDirs[0], symTargetPath)
expectedDirs = [os.path.join(symTargetPath, symTarget), symTargetPath]
expectedFiles = [ os.path.join(self.tempDirs[0], symTarget,"foo.txt"),
os.path.join(self.tempDirs[0], "foo.txt") ]
foundDirs = []
foundFiles = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for dirname in dirs:
foundDirs.append(os.path.join(root, dirname))
for filename in files:
foundFiles.append(os.path.join(root, filename))
self.failUnlessEqual(foundDirs, expectedDirs)
self.failUnlessEqual(foundFiles, expectedFiles)
def testTwoLinksSameTarget(self):
"""
dir1
link1 -> ../dir2
link2 -> ../dir2
dir2
foo.txt
Should follow the symlinks and find foo.txt twice
"""
self.tempDirs.append(tempfile.mkdtemp())
symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w")
symTarget1 = "link1"
symTarget1Path = os.path.join(self.tempDirs[0], symTarget1)
os.symlink(self.tempDirs[1], symTarget1Path)
symTarget2 = "link2"
symTarget2Path = os.path.join(self.tempDirs[0], symTarget2)
os.symlink(self.tempDirs[1], symTarget2Path)
expectedFiles = [
os.path.join(self.tempDirs[0], symTarget2, "foo.txt"),
os.path.join(self.tempDirs[0], symTarget1, "foo.txt")
]
foundFiles = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
foundFiles.append(os.path.join(root, filename))
self.failUnlessEqual(foundFiles, expectedFiles)
def _scanTranslationPath(self, path):
if not os.path.exists(path):
self._console.warn(
"The given path does not contain a translation folder: %s" %
path)
self._console.debug("Scanning translation folder...")
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# Ignore non-script and dot files
if os.path.splitext(
fileName)[-1] != ".po" or fileName.startswith("."):
continue
filePath = os.path.join(root, fileName)
fileLocale = os.path.splitext(fileName)[0]
self._translations[fileLocale] = self.translationEntry(
fileLocale, filePath, self.namespace)
self._console.indent()
self._console.debug("Found %s translations" % len(self._translations))
self._console.outdent()
def _scanResourcePath(self, path):
if not os.path.exists(path):
raise ValueError("The given resource path does not exist: %s" % path)
self._console.debug("Scanning resource folder...")
path = os.path.abspath(path)
lib_prefix_len = len(path)
if not path.endswith(os.sep):
lib_prefix_len += 1
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirectories.match(dir):
dirs.remove(dir)
for file in files:
fpath = os.path.join(root, file)
fpath = os.path.normpath(fpath)
if Image.isImage(fpath):
if CombinedImage.isCombinedImage(fpath):
res = CombinedImage(fpath)
else:
res = Image(fpath)
res.analyzeImage()
else:
res = Resource(fpath)
res.id = Path.posifyPath(fpath[lib_prefix_len:])
res.library= self
self.resources.add(res)
return
def testSymlinkMulti(self):
self.tempDirs.append(tempfile.mkdtemp())
symTarget = os.path.basename(self.tempDirs[1])
symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w")
symTargetPath = os.path.join(self.tempDirs[0], symTarget)
os.symlink(self.tempDirs[1], symTargetPath)
found1 = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
found1.append(filename)
found2 = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
found2.append(filename)
self.failUnlessEqual(found1, found2)
def testSymlinkMulti(self):
self.tempDirs.append(tempfile.mkdtemp())
symTarget = os.path.basename(self.tempDirs[1])
symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w")
symTargetPath = os.path.join(self.tempDirs[0], symTarget)
os.symlink(self.tempDirs[1], symTargetPath)
found1 = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
found1.append(filename)
found2 = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
found2.append(filename)
self.failUnlessEqual(found1, found2)
def testWalkFiles(self):
fooPath = os.path.join(self.tempDirs[0], "foo.txt")
foo = file(fooPath, "w")
foo.close()
barPath = os.path.join(self.tempDirs[0], "bar.txt")
bar = file(barPath, "w")
bar.close()
found = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
found.append(filename)
self.failUnlessEqual(found, ["foo.txt", "bar.txt"], "filetool.walk returned unexpected result %s" %repr(found))
def _getDemos(self):
if not self.hasDemoDir:
return False
demos = {}
demoPath = os.path.join(self.path, "demo")
for root, dirs, files in filetool.walk(demoPath, topdown=True):
for name in dirs:
if root == demoPath and name[0] != "." and name != "source" and name != "build":
foundPath = os.path.join(demoPath, name)
foundDemo = Demo(name, foundPath, self)
demos[name] = foundDemo
return demos
def _getDemos(self):
if not self.hasDemoDir:
return False
demos = {}
demoPath = os.path.join(self.path, "demo")
for root, dirs, files in filetool.walk(demoPath, topdown=True):
for name in dirs:
path = os.path.join(root, name)
manifestPath = os.path.join(path, "Manifest.json")
if os.path.isfile(manifestPath):
foundDemo = Demo(name, path, self)
demos[name] = foundDemo
return demos
def _getDemos(self):
if not self.hasDemoDir:
return False
demos = {}
demoPath = os.path.join(self.path, "demo")
for root, dirs, files in filetool.walk(demoPath, topdown=True):
for name in dirs:
path = os.path.join(root, name)
manifestPath = os.path.join(path, "Manifest.json")
if os.path.isfile(manifestPath):
foundDemo = Demo(name, path, self)
demos[name] = foundDemo
return demos
def _scanResourcePath(self):
resources = set()
if self.resourcePath is None:
return resources
if not os.path.isdir(os.path.join(self.path, self.resourcePath)):
self._console.info("Lib<%s>: Skipping non-existing resource path" %
self.namespace)
return resources
path = os.path.join(self.path, self.resourcePath)
self._console.debug("Scanning resource folder...")
path = os.path.abspath(path)
lib_prefix_len = len(path)
if not path.endswith(os.sep):
lib_prefix_len += 1
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirEntries.match(dir):
dirs.remove(dir)
for file in files:
if self._ignoredDirEntries.match(file):
continue
fpath = os.path.join(root, file)
fpath = os.path.normpath(fpath)
if Image.isImage(fpath):
if CombinedImage.isCombinedImage(fpath):
res = CombinedImage(fpath)
else:
res = Image(fpath)
res.analyzeImage()
elif FontMap.isFontMap(fpath):
res = FontMap(fpath)
else:
res = Resource(fpath)
res.set_id(Path.posifyPath(fpath[lib_prefix_len:]))
res.library = self
resources.add(res)
self._console.indent()
self._console.debug("Found %s resources" % len(resources))
self._console.outdent()
return resources
def testWalkFiles(self):
fooPath = os.path.join(self.tempDirs[0], "foo.txt")
foo = file(fooPath, "w")
foo.close()
barPath = os.path.join(self.tempDirs[0], "bar.txt")
bar = file(barPath, "w")
bar.close()
found = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
found.append(filename)
self.failUnlessEqual(
found, ["foo.txt", "bar.txt"],
"filetool.walk returned unexpected result %s" % repr(found))
def _scanResourcePath(self):
resources = set()
if self.resourcePath is None:
return resources
if not os.path.isdir(os.path.join(self.path,self.resourcePath)):
self._console.info("Lib<%s>: Skipping non-existing resource path" % self.namespace)
return resources
path = os.path.join(self.path,self.resourcePath)
self._console.debug("Scanning resource folder...")
path = os.path.abspath(path)
lib_prefix_len = len(path)
if not path.endswith(os.sep):
lib_prefix_len += 1
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirEntries.match(dir):
dirs.remove(dir)
for file in files:
if self._ignoredDirEntries.match(file):
continue
fpath = os.path.join(root, file)
fpath = os.path.normpath(fpath)
if Image.isImage(fpath):
if CombinedImage.isCombinedImage(fpath):
res = CombinedImage(fpath)
else:
res = Image(fpath)
res.analyzeImage()
elif FontMap.isFontMap(fpath):
res = FontMap(fpath)
else:
res = Resource(fpath)
res.set_id(Path.posifyPath(fpath[lib_prefix_len:]))
res.library= self
resources.add(res)
self._console.indent()
self._console.debug("Found %s resources" % len(resources))
self._console.outdent()
return resources
def classPathIterator(self):
if self.classPath is None:
return
classRoot = os.path.join(self.path, self.classPath)
for root, dirs, files in filetool.walk(classRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(fileName):
continue
filePath = os.path.join(root, fileName)
filePathId = filePath.replace(classRoot + os.sep, '')
yield (filePathId, filePath)
def _scanResourcePath(self, path):
if not os.path.exists(path):
raise ValueError("The given resource path does not exist: %s" % path)
self._console.debug("Scanning resource folder...")
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirectories.match(dir):
dirs.remove(dir)
for file in files:
fpath = os.path.join(root, file)
#self._resources.append(fpath) # avoiding this currently, as it is not used
if CombinedImage.isCombinedImage(fpath):
self.resources.combImages.add(os.path.normpath(fpath))
return
def classPathIterator(self):
if self.classPath is None:
return
classRoot = os.path.join(self.path, self.classPath)
for root, dirs, files in filetool.walk(classRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(
fileName):
continue
filePath = os.path.join(root, fileName)
filePathId = filePath.replace(classRoot + os.sep, '')
yield (filePathId, filePath)
def _scanResourcePath(self, path):
if not os.path.exists(path):
raise ValueError("The given resource path does not exist: %s" %
path)
self._console.debug("Scanning resource folder...")
path = os.path.abspath(path)
lib_prefix_len = len(path)
if not path.endswith(os.sep):
lib_prefix_len += 1
self.resources = set()
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirEntries.match(dir):
dirs.remove(dir)
for file in files:
if self._ignoredDirEntries.match(file):
continue
fpath = os.path.join(root, file)
fpath = os.path.normpath(fpath)
if Image.isImage(fpath):
if CombinedImage.isCombinedImage(fpath):
res = CombinedImage(fpath)
else:
res = Image(fpath)
res.analyzeImage()
else:
res = Resource(fpath)
res.set_id(Path.posifyPath(fpath[lib_prefix_len:]))
res.library = self
self.resources.add(res)
self._console.indent()
self._console.debug("Found %s resources" % len(self.resources))
self._console.outdent()
return
def _scanResourcePath(self, path):
if not os.path.exists(path):
raise ValueError("The given resource path does not exist: %s" %
path)
self._console.debug("Scanning resource folder...")
for root, dirs, files in filetool.walk(path):
# filter ignored directories
for dir in dirs:
if self._ignoredDirectories.match(dir):
dirs.remove(dir)
for file in files:
fpath = os.path.join(root, file)
#self._resources.append(fpath) # avoiding this currently, as it is not used
if CombinedImage.isCombinedImage(fpath):
self.resources.combImages.add(fpath)
return
def testSymlink(self):
"""
dir1
dir2 -> ../dir2
dir2
foo.txt
Should follow the symlink and find foo.txt
"""
self.tempDirs.append(tempfile.mkdtemp())
symTarget = os.path.basename(self.tempDirs[1])
symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w")
symTargetPath = os.path.join(self.tempDirs[0], symTarget)
os.symlink(self.tempDirs[1], symTargetPath)
found = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
found.append(filename)
self.failUnlessEqual(found, ["foo.txt"])
def _scanTranslationPath(self):
translations = {} # reset
if self.assets['translations']['path'] is None:
return translations
if not os.path.isdir(
os.path.join(self.path, self.assets['translations']['path'])):
self._console.info(
"Lib<%s>: Skipping non-existing translation path" %
self.namespace)
return translations
path = os.path.join(self.path, self.assets['translations']['path'])
self._console.debug("Scanning translation folder...")
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# Ignore non-po and dot files
if os.path.splitext(
fileName)[-1] != ".po" or fileName.startswith("."):
continue
filePath = os.path.join(root, fileName)
fileLocale = os.path.splitext(fileName)[0]
translations[fileLocale] = self.translationEntry(
fileLocale, filePath, self.namespace)
self._console.indent()
self._console.debug("Found %s translations" % len(translations))
self._console.outdent()
return translations
def testSymlink(self):
"""
dir1
dir2 -> ../dir2
dir2
foo.txt
Should follow the symlink and find foo.txt
"""
self.tempDirs.append(tempfile.mkdtemp())
symTarget = os.path.basename(self.tempDirs[1])
symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w")
symTargetPath = os.path.join(self.tempDirs[0], symTarget)
os.symlink(self.tempDirs[1], symTargetPath)
found = []
for root, dirs, files in filetool.walk(self.tempDirs[0]):
for filename in files:
found.append(filename)
self.failUnlessEqual(found, ["foo.txt"])
def _scanClassPath(self, timeOfLastScan=0):
codeIdFromTree = True # switch between regex- and tree-based codeId search
# Check class path
classPath = os.path.join(self.path, self.classPath)
if not os.path.isdir(classPath):
raise ConfigurationError("Class path from Manifest doesn't exist: %s" % self.classPath)
# Check multiple namespaces
if not len([d for d in os.listdir(classPath) if not d.startswith(".")]) == 1:
self._console.warn(
"The class path must contain exactly one namespace; ignoring everything else: '%s'" % (classPath,)
)
# Check Manifest namespace matches file system
nsPrefix = self.namespace.replace(".", os.sep)
classNSRoot = os.path.join(classPath, nsPrefix)
if not os.path.isdir(classNSRoot):
raise ValueError("Manifest namespace does not exist on file system: '%s'" % (classNSRoot))
self._console.debug("Scanning class folder...")
classList = []
existClassIds = dict([(x.id, x) for x in self._classes]) # if we scanned before
docs = {}
# TODO: Clazz still relies on a context dict!
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
# Iterate...
for root, dirs, files in filetool.walk(classNSRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Add good directories
currNameSpace = root[len(classNSRoot + os.sep) :]
currNameSpace = currNameSpace.replace(os.sep, ".")
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(fileName):
continue
self._console.dot()
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(classNSRoot + os.sep, "")
fileExt = os.path.splitext(fileName)[-1]
fileStat = os.stat(filePath)
fileSize = fileStat.st_size
fileMTime = fileStat.st_mtime
# Compute full URI from relative path
fileUri = self.classUri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
filePathId = self.namespace + "." + filePathId
filePathId = unidata.normalize("NFC", filePathId) # combine combining chars: o" -> ö
# check if known and fresh
if filePathId in existClassIds and fileMTime < timeOfLastScan:
classList.append(existClassIds[filePathId])
# print "re-using existing", filePathId
continue # re-use known class
# Extract package ID
filePackage = filePathId[: filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[: filePathId.rfind(".")]
docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": self.encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize,
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict, self._classesObj)
else:
clazz = Class(filePathId, filePath, self, contextdict, self._classesObj)
# Extract code ID (e.g. class name, mixin name, ...)
try:
if codeIdFromTree:
fileCodeId = self._getCodeId(clazz)
else:
# Read content
fileContent = filetool.read(filePath, self.encoding)
fileCodeId = self._getCodeId1(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u" (%s)" % fileName
e.args = tuple(argsList)
raise e
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error("Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("Path: %s" % fileRel)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
clazz.encoding = self.encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
clazz.relpath = fileRel # Locale uses this
clazz.m_time_ = fileStat.st_mtime
classList.append(clazz)
def _scanClassPath(self, timeOfLastScan=0):
##
# provide a default context dict
def get_contextdict():
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
return contextdict
##
# check class path is on file system
def check_class_path(classRoot):
if not os.path.isdir(classRoot):
raise ConfigurationError("Class path from Manifest doesn't exist: %s" % self.classPath)
##
# check single subdirectory from class path
def check_multiple_namespaces(classRoot):
if not len([d for d in os.listdir(classRoot) if not d.startswith(".")]) == 1:
self._console.warn("The class path should contain exactly one namespace: '%s'" % (classRoot,))
##
# check manifest namespace is on file system
def check_manifest_namespace(classRoot):
nsPrefix = self.namespace.replace(".", os.sep)
classNSRoot = os.path.join(classRoot, nsPrefix)
if not os.path.isdir(classNSRoot):
raise ValueError("Manifest namespace does not exist on file system: '%s'" % (classNSRoot))
# ----------------------------------------------------------------------
codeIdFromTree = True # switch between regex- and tree-based codeId search
classList = []
existClassIds = dict([(x.id, x) for x in self._classes]) # if we scanned before
docs = {}
contextdict = get_contextdict() # TODO: Clazz() still relies on a context dict!
classRoot = os.path.join(self.path, self.classPath)
check_class_path(classRoot)
check_multiple_namespaces(classRoot)
check_manifest_namespace(classRoot)
self._console.debug("Scanning class folder...")
# Iterate...
for root, dirs, files in filetool.walk(classRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(fileName):
continue
self._console.dot()
# basic attributes
filePath = os.path.join(root, fileName) # /foo/bar/baz/source/class/my/space/AppClass.js
fileRel = filePath.replace(classRoot + os.sep, "") # my/space/AppClass.js
fileExt = os.path.splitext(fileName)[-1] # "js"
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".") # my.space.AppClass
filePathId = unidata.normalize("NFC", filePathId) # o" -> ö
filePackage = filePathId[: filePathId.rfind(".")] if "." in filePathId else "" # my.space
fileStat = os.stat(filePath)
fileSize = fileStat.st_size
fileMTime = fileStat.st_mtime
# ignore non-script
if fileExt != ".js":
continue
# check if known and fresh
if filePathId in existClassIds and fileMTime < timeOfLastScan:
classList.append(existClassIds[filePathId])
continue # re-use known class
# handle doc files
if fileName == self._docFilename:
docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": self.encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize,
}
# stop further processing
continue
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict)
else:
clazz = Class(filePathId, filePath, self, contextdict)
# extract code ID (e.g. class name, mixin name, ...)
try:
if codeIdFromTree:
fileCodeId = self._getCodeId(clazz)
else:
# use regexp
fileContent = filetool.read(filePath, self.encoding)
fileCodeId = self._getCodeId1(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u" (%s)" % fileName
e.args = tuple(argsList)
raise e
# ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# compare path and content
if fileCodeId != filePathId:
errmsg = [
u"Detected conflict between filename and classname!\n",
u" Classname: %s\n" % fileCodeId,
u" Path: %s\n" % filePath,
]
raise RuntimeError(u"".join(errmsg))
# store file data
self._console.debug("Adding class %s" % filePathId)
clazz.encoding = self.encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
clazz.relpath = fileRel # Locale uses this
clazz.m_time_ = fileStat.st_mtime
classList.append(clazz)
def _scanClassPath(self, path, uri, encoding):
if not os.path.exists(path):
raise ValueError("The given class path does not exist: %s" % path)
self._console.debug("Scanning class folder...")
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirectories.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith("."):
continue
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(path + os.sep, "")
fileExt = os.path.splitext(fileName)[-1]
fileSize = os.stat(filePath).st_size
# Compute full URI from relative path
fileUri = uri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
# Extract package ID
filePackage = filePathId[:filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[:filePathId.rfind(".")]
self._docs[filePackage] = {
"relpath" : fileRel,
"path" : filePath,
"encoding" : encoding,
"namespace" : self._namespace,
"id" : filePathId,
"package" : filePackage,
"size" : fileSize
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
# Read content
fileContent = filetool.read(filePath, encoding)
# Extract code ID (e.g. class name, mixin name, ...)
try:
fileCodeId = self._getCodeId(fileContent)
except ValueError, e:
raise ValueError, e.message + u' (%s)' % fileName
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error("Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("Path: %s" % fileRel)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
self._classes[filePathId] = {
"relpath" : fileRel,
"path" : filePath,
"encoding" : encoding,
"namespace" : self._namespace,
"id" : filePathId,
"package" : filePackage,
"size" : fileSize
}
def check_multiple_namespaces(classRoot):
for root, dirs, files in filetool.walk(classRoot):
if len(dirs) != 1:
self._console.warn ("The class path should contain exactly one namespace: '%s'" % (classRoot,))
break
def _scanClassPath(self, path, uri, encoding):
if not os.path.exists(path):
raise ValueError("The given class path does not exist: %s" % path)
self._console.debug("Scanning class folder...")
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirectories.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith("."):
continue
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(path + os.sep, "")
fileExt = os.path.splitext(fileName)[-1]
fileSize = os.stat(filePath).st_size
# Compute full URI from relative path
fileUri = uri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
# Extract package ID
filePackage = filePathId[:filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[:filePathId.rfind(".")]
self._docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": encoding,
"namespace": self._namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
# Read content
fileContent = filetool.read(filePath, encoding)
# Extract code ID (e.g. class name, mixin name, ...)
try:
fileCodeId = self._getCodeId(fileContent)
except ValueError, e:
raise ValueError, e.message + u' (%s)' % fileName
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error(
"Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("Path: %s" % fileRel)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
self._classes[filePathId] = {
"relpath": fileRel,
"path": filePath,
"encoding": encoding,
"namespace": self._namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
def _scanClassPath(self, timeOfLastScan=0):
codeIdFromTree = True # switch between regex- and tree-based codeId search
# Check class path
classPath = os.path.join(self.path, self.classPath)
if not os.path.isdir(classPath):
raise ConfigurationError(
"Class path from Manifest doesn't exist: %s" % self.classPath)
# Check multiple namespaces
if not len([d for d in os.listdir(classPath) if not d.startswith(".")
]) == 1:
self._console.warn(
"The class path must contain exactly one namespace; ignoring everything else: '%s'"
% (classPath, ))
# Check Manifest namespace matches file system
nsPrefix = self.namespace.replace(".", os.sep)
classNSRoot = os.path.join(classPath, nsPrefix)
if not os.path.isdir(classNSRoot):
raise ValueError(
"Manifest namespace does not exist on file system: '%s'" %
(classNSRoot))
self._console.debug("Scanning class folder...")
classList = []
existClassIds = dict([(x.id, x)
for x in self._classes]) # if we scanned before
docs = {}
# TODO: Clazz still relies on a context dict!
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
# Iterate...
for root, dirs, files in filetool.walk(classNSRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Add good directories
currNameSpace = root[len(classNSRoot + os.sep):]
currNameSpace = currNameSpace.replace(os.sep,
".") # TODO: var name
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(
fileName):
continue
self._console.dot()
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(
classNSRoot + os.sep,
"") # now only path fragment *afte* NS
fileExt = os.path.splitext(fileName)[-1]
fileStat = os.stat(filePath)
fileSize = fileStat.st_size
fileMTime = fileStat.st_mtime
# Compute full URI from relative path
fileUri = self.classUri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
filePathId = self.namespace + "." + filePathId # e.g. "qx.core.Environment"
filePathId = unidata.normalize(
"NFC", filePathId) # combine combining chars: o" -> ö
fileId = nsPrefix + "/" + fileRel # e.g. "qx/core/Environment.js"
# check if known and fresh
if (filePathId in existClassIds
and fileMTime < timeOfLastScan):
classList.append(existClassIds[filePathId])
#print "re-using existing", filePathId
continue # re-use known class
# Extract package ID
filePackage = filePathId[:filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[:filePathId.rfind(".")]
docs[filePackage] = {
"relpath": fileId,
"path": filePath,
"encoding": self.encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict)
else:
clazz = Class(filePathId, filePath, self, contextdict)
# Extract code ID (e.g. class name, mixin name, ...)
try:
if codeIdFromTree:
fileCodeId = self._getCodeId(clazz)
else:
# Read content
fileContent = filetool.read(filePath, self.encoding)
fileCodeId = self._getCodeId1(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u' (%s)' % fileName
e.args = tuple(argsList)
raise e
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error(
"Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("Path: %s" % filePath)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
clazz.encoding = self.encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
clazz.relpath = fileId # Locale uses this
clazz.m_time_ = fileStat.st_mtime
classList.append(clazz)
def _scanClassPath(self, timeOfLastScan=0):
##
# provide a default context dict
def get_contextdict():
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
contextdict["envchecksmap"] = {}
return contextdict
##
# check class path is on file system
def check_class_path(classRoot):
if not os.path.isdir(classRoot):
raise ConfigurationError(
"Class path from Manifest doesn't exist: %s" %
self.classPath)
##
# check single subdirectory from class path
def check_multiple_namespaces(classRoot):
try:
self._checkNamespace(classRoot)
except ValueError:
self._console.warn(
"The class path should contain exactly one namespace: '%s'"
% (classRoot, ))
##
# check manifest namespace is on file system
def check_manifest_namespace(classRoot):
nsPrefix = self.namespace.replace(".", os.sep)
classNSRoot = os.path.join(classRoot, nsPrefix)
if not os.path.isdir(classNSRoot):
raise ValueError(
"Manifest namespace does not exist on file system: '%s'" %
(classNSRoot))
# ----------------------------------------------------------------------
codeIdFromTree = True # switch between regex- and tree-based codeId search
classList = []
existClassIds = dict([(x.id, x)
for x in self._classes]) # if we scanned before
docs = {}
contextdict = get_contextdict(
) # TODO: Clazz() still relies on a context dict!
classRoot = os.path.join(self.path, self.classPath)
check_class_path(classRoot)
check_multiple_namespaces(classRoot)
check_manifest_namespace(classRoot)
self._console.debug("Scanning class folder...")
# Iterate...
for root, dirs, files in filetool.walk(classRoot):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirEntries.match(ignoredDir):
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
# ignore dot files
if fileName.startswith(".") or self._ignoredDirEntries.match(
fileName):
continue
self._console.dot()
# basic attributes
filePath = os.path.join(
root,
fileName) # /foo/bar/baz/source/class/my/space/AppClass.js
fileRel = filePath.replace(classRoot + os.sep,
'') # my/space/AppClass.js
fileExt = os.path.splitext(fileName)[-1] # "js"
filePathId = fileRel.replace(fileExt, "").replace(
os.sep, ".") # my.space.AppClass
filePathId = unidata.normalize("NFC", filePathId) # o" -> ö
filePackage = filePathId[:filePathId.rfind(
".")] if "." in filePathId else "" # my.space
fileStat = os.stat(filePath)
fileSize = fileStat.st_size
fileMTime = fileStat.st_mtime
# ignore non-script
if fileExt != ".js":
continue
# check if known and fresh
if (filePathId in existClassIds
and fileMTime < timeOfLastScan):
classList.append(existClassIds[filePathId])
continue # re-use known class
# handle doc files
if fileName == self._docFilename:
docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": self.encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# stop further processing
continue
if filePathId == "qx.core.Environment":
clazz = qcEnvClass(filePathId, filePath, self, contextdict)
else:
clazz = Class(filePathId, filePath, self, contextdict)
# extract code ID (e.g. class name, mixin name, ...)
try:
if codeIdFromTree:
fileCodeId = self._getCodeId(clazz)
else:
# use regexp
fileContent = filetool.read(filePath, self.encoding)
fileCodeId = self._getCodeId1(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u' (%s)' % fileName
e.args = tuple(argsList)
raise e
# ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# compare path and content
if fileCodeId != filePathId:
errmsg = [
u"Detected conflict between filename and classname!\n",
u" Classname: %s\n" % fileCodeId,
u" Path: %s\n" % filePath,
]
raise RuntimeError(u''.join(errmsg))
# store file data
self._console.debug("Adding class %s" % filePathId)
clazz.encoding = self.encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
clazz.relpath = fileRel # Locale uses this
clazz.m_time_ = fileStat.st_mtime
classList.append(clazz)
def _scanClassPath(self, path, uri, encoding):
if not os.path.exists(path):
raise ValueError("The given class path does not exist: %s" % path)
self._console.debug("Scanning class folder...")
classList = {}
docs = {}
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirectories.match(ignoredDir):
dirs.remove(ignoredDir)
# Add good directories
currNameSpace = root[len(path+os.sep):]
currNameSpace = currNameSpace.replace(os.sep, ".")
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith("."):
continue
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(path + os.sep, "")
fileExt = os.path.splitext(fileName)[-1]
fileSize = os.stat(filePath).st_size
# Compute full URI from relative path
fileUri = uri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
# Extract package ID
filePackage = filePathId[:filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[:filePathId.rfind(".")]
docs[filePackage] = {
"relpath" : fileRel,
"path" : filePath,
"encoding" : encoding,
"namespace" : self.namespace,
"id" : filePathId,
"package" : filePackage,
"size" : fileSize
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
# Read content
fileContent = filetool.read(filePath, encoding)
# Extract code ID (e.g. class name, mixin name, ...)
try:
fileCodeId = self._getCodeId(fileContent)
except ValueError, e:
argsList = []
for arg in e.args:
argsList.append(arg)
argsList[0] = argsList[0] + u' (%s)' % fileName
e.args = tuple(argsList)
raise e
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error("Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("Path: %s" % fileRel)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
classList[filePathId] = {
"relpath" : fileRel,
"path" : filePath,
"encoding" : encoding,
"namespace" : self.namespace,
"id" : filePathId,
"package" : filePackage,
"size" : fileSize
}
# TODO: Clazz still relies on a context dict!
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
# TODO: currently creation of throw-away objects (unless they're .append'ed)
clazz = Class(classList[filePathId], filePath, self, contextdict, self._classesObj)
clazz.encoding = encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this
def _scanClassPath(self, path, uri, encoding):
if not os.path.exists(path):
raise ValueError("The given class path does not exist: %s" % path)
self._console.debug("Scanning class folder...")
classList = {}
docs = {}
# Iterate...
for root, dirs, files in filetool.walk(path):
# Filter ignored directories
for ignoredDir in dirs:
if self._ignoredDirectories.match(ignoredDir):
dirs.remove(ignoredDir)
# Add good directories
currNameSpace = root[len(path + os.sep):]
currNameSpace = currNameSpace.replace(os.sep, ".")
# Searching for files
for fileName in files:
# Ignore dot files
if fileName.startswith("."):
continue
# Process path data
filePath = os.path.join(root, fileName)
fileRel = filePath.replace(path + os.sep, "")
fileExt = os.path.splitext(fileName)[-1]
fileSize = os.stat(filePath).st_size
# Compute full URI from relative path
fileUri = uri + "/" + fileRel.replace(os.sep, "/")
# Compute identifier from relative path
filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")
# Extract package ID
filePackage = filePathId[:filePathId.rfind(".")]
# Handle doc files
if fileName == self._docFilename:
fileFor = filePathId[:filePathId.rfind(".")]
docs[filePackage] = {
"relpath": fileRel,
"path": filePath,
"encoding": encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# Stop further processing
continue
# Ignore non-script
if os.path.splitext(fileName)[-1] != ".js":
continue
# Read content
fileContent = filetool.read(filePath, encoding)
# Extract code ID (e.g. class name, mixin name, ...)
try:
fileCodeId = self._getCodeId(fileContent)
except ValueError, e:
e.args[0] = e.args[0] + u' (%s)' % fileName
raise e
# Ignore all data files (e.g. translation, doc files, ...)
if fileCodeId == None:
continue
# Compare path and content
if fileCodeId != filePathId:
self._console.error(
"Detected conflict between filename and classname!")
self._console.indent()
self._console.error("Classname: %s" % fileCodeId)
self._console.error("filePathId: %s" % filePathId)
self._console.error("Path: %s" % fileRel)
self._console.outdent()
raise RuntimeError()
# Store file data
self._console.debug("Adding class %s" % filePathId)
classList[filePathId] = {
"relpath": fileRel,
"path": filePath,
"encoding": encoding,
"namespace": self.namespace,
"id": filePathId,
"package": filePackage,
"size": fileSize
}
# TODO: Clazz still relies on a context dict!
contextdict = {}
contextdict["console"] = context.console
contextdict["cache"] = context.cache
contextdict["jobconf"] = context.jobconf
# TODO: currently creation of throw-away objects (unless they're .append'ed)
clazz = Class(classList[filePathId], filePath, self,
contextdict, self._classesObj)
clazz.encoding = encoding
clazz.size = fileSize # dependency logging uses this
clazz.package = filePackage # Apiloader uses this