def store(self, key, value, timestamp=None, transient=False, inMemory=True):
"""
Stores the given value.
Default timestamp goes to the current time. Can be modified
to the time of an other files modification date etc.
Transient enables in-memory cache for the given value
"""
if self.__hashkeys:
key = hashlib.sha1(key.encode("ascii")).hexdigest()
if inMemory:
self.__transient[key] = value
if transient:
return
if not timestamp:
timestamp = time.time()
try:
self.__shelve[key + "-timestamp"] = timestamp
self.__shelve[key] = value
except pickle.PicklingError as err:
Console.error("Failed to store enty: %s" % key)
def watch(path, callback):
"""
Start observing changes in filesystem. See JasyEventHandler for the event callbacks.
:param path: Path wich will be observed
:type name: string
"""
if Observer is None:
Console.error("You need to install Watchdog for supporting file system watchers")
# Initialize file system observer
observer = Observer()
observer.schedule(JasyEventHandler(), ".", recursive=True)
observer.start()
Console.info("Started file system watcher for %s... [PID=%s]", path, os.getpid())
Console.info("Use 'ulimit -n 1024' to increase number of possible open files")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
Console.info("Stopped file system watcher for %s...", path)
observer.join()
def init(self, autoInitialize=True, updateRepositories=True, scriptEnvironment=None):
"""
Initialize the actual session with projects
:param autoInitialize: Whether the projects should be automatically added when the current folder contains a valid Jasy project.
:param updateRepositories: Whether to update repositories of all project dependencies.
:param scriptEnvironment: API object as being used for loadLibrary to add Python features offered by projects.
"""
self.__scriptEnvironment = scriptEnvironment
self.__updateRepositories = updateRepositories
if autoInitialize and jasy.core.Config.findConfig("jasyproject"):
try:
self.addProject(jasy.core.Project.getProjectFromPath("."))
except UserError as err:
Console.outdent(True)
Console.error(err)
raise UserError("Critical: Could not initialize session!")
Console.info("Active projects (%s):", len(self.__projects))
Console.indent()
for project in self.__projects:
if project.version:
Console.info("%s @ %s", Console.colorize(project.getName(), "bold"), Console.colorize(project.version, "magenta"))
else:
Console.info(Console.colorize(project.getName(), "bold"))
Console.outdent()
def getApi(className):
classApi = apiData[className]
if className in mergedClasses:
return classApi
classIncludes = getattr(classApi, "includes", None)
if classIncludes:
for mixinName in classIncludes:
if mixinName not in apiData:
Console.error("Invalid mixin %s in class %s",
className, mixinName)
continue
mixinApi = apiData[mixinName]
if not hasattr(mixinApi, "includedBy"):
mixinApi.includedBy = set()
mixinApi.includedBy.add(className)
mergeMixin(className, mixinName, classApi,
getApi(mixinName))
mergedClasses.add(className)
return classApi
def store(self,
key,
value,
timestamp=None,
transient=False,
inMemory=True):
"""
Stores the given value.
Default timestamp goes to the current time. Can be modified
to the time of an other files modification date etc.
Transient enables in-memory cache for the given value
"""
if self.__hashkeys:
key = hashlib.sha1(key.encode("ascii")).hexdigest()
if inMemory:
self.__transient[key] = value
if transient:
return
if not timestamp:
timestamp = time.time()
try:
self.__shelve[key + "-timestamp"] = timestamp
self.__shelve[key] = value
except pickle.PicklingError as err:
Console.error("Failed to store enty: %s" % key)
def test(target="source",
tool="phantom",
browsers=None,
main="test.Main",
kernel="test.Kernel"):
"""
Automatically executes tests in either PhantomJS, NodeJS or via Testem CI
"""
# Initialize profile
profile = Profile.Profile(session)
profile.registerPart("kernel", className=kernel)
profile.registerPart("main", className=main)
# Destination should match target name
profile.setDestinationPath(target)
if target == "source":
# Force debug enabled
profile.setField("debug", True)
# Load all scripts/assets from source folder
profile.setUseSource(True)
# Start actual build
Build.run(profile)
elif target == "build":
# Copy assets
profile.setCopyAssets(True)
# Start actual build
Build.run(profile)
# Copy files from source
for name in ["index.html", "testem.html", "phantom.js", "node.js"]:
fileManager.updateFile("source/%s" % fileName,
"{{destination}}/%s" % fileName)
else:
Console.error("Unsupported target: %s" % target)
if tool == "phantom":
return test_phantom(profile)
elif tool == "node":
return test_node(profile)
elif tool == "testem":
return test_testem(profile, target, browsers)
else:
Console.error("Unsupported tool: %s" % tool)
def processInternalLink(match):
linkUrl = match.group(2)
if linkUrl.startswith("#"):
linkCheck = checkInternalLink(linkUrl[1:], className)
if linkCheck is not True:
item["errornous"] = True
if sectionName:
Console.error("%s in %s:%s~%s" % (linkCheck, sectionName, className, name))
else:
Console.error("%s in %s" % (linkCheck, className))
def executeTask(taskname, **kwargs):
"""Executes the given task by name with any optional named arguments."""
if taskname in __taskRegistry:
try:
camelCaseArgs = {Util.camelize(key) : kwargs[key] for key in kwargs}
return __taskRegistry[taskname](**camelCaseArgs)
except UserError as err:
raise
except:
Console.error("Unexpected error! Could not finish task %s successfully!" % taskname)
raise
else:
raise UserError("No such task: %s" % taskname)
def processInternalLink(match):
linkUrl = match.group(2)
if linkUrl.startswith("#"):
linkCheck = checkInternalLink(linkUrl[1:], className)
if linkCheck is not True:
item["errornous"] = True
if sectionName:
Console.error(
"%s in %s:%s~%s" %
(linkCheck, sectionName, className, name))
else:
Console.error("%s in %s" %
(linkCheck, className))
def open(self):
"""Opens a cache file in the given path."""
try:
self.__shelve = shelve.open(self.__file, flag="c")
storedVersion = jasy.core.Util.getKey(self.__shelve,
"jasy-version")
storedHost = jasy.core.Util.getKey(self.__shelve, "jasy-host")
if storedVersion == jasy.__version__ and storedHost == hostId:
return
if storedVersion is not None or storedHost is not None:
Console.debug(
"Jasy version or host has been changed. Recreating cache..."
)
self.clear()
self.__shelve = shelve.open(self.__file, flag="n")
self.__shelve["jasy-version"] = jasy.__version__
self.__shelve["jasy-host"] = hostId
except dbm.error as dbmerror:
errno = None
try:
errno = dbmerror.errno
except:
pass
if errno is 35:
raise IOError("Cache file is locked by another process!")
elif "type could not be determined" in str(dbmerror):
Console.error("Could not detect cache file format: %s" %
self.__file)
Console.warn("Recreating cache database...")
self.clear()
elif "module is not available" in str(dbmerror):
Console.error("Unsupported cache file format: %s" %
self.__file)
Console.warn("Recreating cache database...")
self.clear()
else:
raise dbmerror
def test(target="source", tool="phantom", browsers=None, main="test.Main", kernel="test.Kernel"):
"""
Automatically executes tests in either PhantomJS, NodeJS or via Testem CI
"""
# Initialize profile
profile = Profile.Profile(session)
profile.registerPart("kernel", className=kernel)
profile.registerPart("main", className=main)
# Destination should match target name
profile.setDestinationPath(target)
if target == "source":
# Force debug enabled
profile.setField("debug", True)
# Load all scripts/assets from source folder
profile.setUseSource(True)
# Start actual build
Build.run(profile)
elif target == "build":
# Copy assets
profile.setCopyAssets(True)
# Start actual build
Build.run(profile)
# Copy files from source
for name in ["index.html", "testem.html", "phantom.js", "node.js"]:
fileManager.updateFile("source/%s" % fileName, "{{destination}}/%s" % fileName)
else:
Console.error("Unsupported target: %s" % target)
if tool == "phantom":
return test_phantom(profile)
elif tool == "node":
return test_node(profile)
elif tool == "testem":
return test_testem(profile, target, browsers)
else:
Console.error("Unsupported tool: %s" % tool)
def copyFile(self, src, dst):
"""Copy src file to dst file. Both should be filenames, not directories."""
if not os.path.isfile(src):
raise Exception("No such file: %s" % src)
dst = self.__session.expandFileName(dst)
# First test for existance of destination directory
self.makeDir(os.path.dirname(dst))
# Finally copy file to directory
try:
shutil.copy2(src, dst)
except IOError as ex:
Console.error("Could not write file %s: %s" % (dst, ex))
return True
def open(self):
"""Opens a cache file in the given path."""
try:
self.__shelve = shelve.open(self.__file, flag="c")
storedVersion = jasy.core.Util.getKey(self.__shelve, "jasy-version")
storedHost = jasy.core.Util.getKey(self.__shelve, "jasy-host")
if storedVersion == jasy.__version__ and storedHost == hostId:
return
if storedVersion is not None or storedHost is not None:
Console.debug("Jasy version or host has been changed. Recreating cache...")
self.clear()
self.__shelve = shelve.open(self.__file, flag="n")
self.__shelve["jasy-version"] = jasy.__version__
self.__shelve["jasy-host"] = hostId
except dbm.error as dbmerror:
errno = None
try:
errno = dbmerror.errno
except:
pass
if errno is 35:
raise IOError("Cache file is locked by another process!")
elif "type could not be determined" in str(dbmerror):
Console.error("Could not detect cache file format: %s" % self.__file)
Console.warn("Recreating cache database...")
self.clear()
elif "module is not available" in str(dbmerror):
Console.error("Unsupported cache file format: %s" % self.__file)
Console.warn("Recreating cache database...")
self.clear()
else:
raise dbmerror
def test(target="source", tool="phantom", browsers=None, main="test.Main"):
"""Automatically executes tests in either PhantomJS, NodeJS or Testem CI"""
session.setCurrentPrefix(target)
if target == "source":
test_source(main=main)
elif target == "build":
test_build(main=main)
else:
Console.error("Unsupported target: %s" % target)
if tool == "phantom":
test_phantom()
elif tool == "node":
test_node()
elif tool == "testem":
test_testem(browsers)
else:
Console.error("Unsupported tool: %s" % tool)
def normalizePackageList(pkgs):
res = []
for pkg in pkgs:
if isinstance(pkg, str):
res.append((pkg, None, None))
else:
version = None
source = None
if len(pkg) == 1:
(name) = pkg
elif len(pkg) == 2:
(name, version) = pkg
elif len(pkg) == 3:
(name, version, source) = pkg
else:
Console.error("Package declaration wrong, should be (packagename), (packagename, version) or (packagename, version, source)")
res.append((name, version, source))
return res
def init(self, autoInitialize=True, updateRepositories=True, scriptEnvironment=None):
"""
Initialize the actual session with projects.
:param autoInitialize: Whether the projects should be automatically added when the current folder contains a valid Jasy project.
:param updateRepositories: Whether to update repositories of all project dependencies.
:param scriptEnvironment: API object as being used for loadLibrary to add Python features offered by projects.
:param commandEnvironment: API object as being used for loadCommands to add Python features for any item nodes.
"""
self.__scriptEnvironment = scriptEnvironment
self.__updateRepositories = updateRepositories
if autoInitialize and Config.findConfig("jasyproject"):
Console.info("Initializing session...")
Console.indent()
try:
self.addProject(Project.getProjectFromPath(".", self))
except UserError as err:
Console.outdent(True)
Console.error(err)
raise UserError("Critical: Could not initialize session!")
self.getVirtualProject()
Console.debug("Active projects (%s):", len(self.__projects))
Console.indent()
for project in self.__projects:
if project.version:
Console.debug("%s @ %s", Console.colorize(project.getName(), "bold"), Console.colorize(project.version, "magenta"))
else:
Console.debug(Console.colorize(project.getName(), "bold"))
Console.outdent()
Console.outdent()
def parse(self, path, languages):
Console.info("Processing %s..." % path)
Console.indent()
collection = []
for extension in self.__extensions:
for filename in glob.iglob(os.path.join(path, "*.%s" % extension)):
basename = os.path.basename(filename)
Console.debug("Parsing %s" % basename)
parsed = self.__parseWithSpecificParser(filename, extension)
if not parsed:
Console.error("Error parsing file %s" % filename)
continue
self.postProcess(parsed, filename, languages)
collection.append(parsed)
Console.info("Registered %s files.", len(collection))
Console.outdent()
return collection
def getApi(className):
classApi = apiData[className]
if className in mergedClasses:
return classApi
classIncludes = getattr(classApi, "includes", None)
if classIncludes:
for mixinName in classIncludes:
if not mixinName in apiData:
Console.error("Invalid mixin %s in class %s", className, mixinName)
continue
mixinApi = apiData[mixinName]
if not hasattr(mixinApi, "includedBy"):
mixinApi.includedBy = set()
mixinApi.includedBy.add(className)
mergeMixin(className, mixinName, classApi, getApi(mixinName))
mergedClasses.add(className)
return classApi
def copyFile(self, src, dst):
"""
Copy src file to dst file.
Both should be filenames, not directories.
"""
if not os.path.isfile(src):
raise Exception("No such file: %s" % src)
if self.__profile:
dst = self.__profile.expandFileName(dst)
# First test for existance of destination directory
self.makeDir(os.path.dirname(dst))
# Finally copy file to directory
try:
shutil.copy2(src, dst)
except IOError as ex:
Console.error("Could not write file %s: %s" % (dst, ex))
return True
def checkLinksInItem(item):
# Process types
if "type" in item:
if item["type"] == "Function":
# Check param types
if "params" in item:
for paramName in item["params"]:
paramEntry = item["params"][paramName]
if "type" in paramEntry:
for paramTypeEntry in paramEntry["type"]:
if not paramTypeEntry["name"] in knownClasses and not paramTypeEntry["name"] in additionalTypes and not ("builtin" in paramTypeEntry or "pseudo" in paramTypeEntry):
item["errornous"] = True
Console.error('Invalid param type "%s" in %s' % (paramTypeEntry["name"], className))
if not "pseudo" in paramTypeEntry and paramTypeEntry["name"] in knownClasses:
paramTypeEntry["linkable"] = True
# Check return types
if "returns" in item:
for returnTypeEntry in item["returns"]:
if not returnTypeEntry["name"] in knownClasses and not returnTypeEntry["name"] in additionalTypes and not ("builtin" in returnTypeEntry or "pseudo" in returnTypeEntry):
item["errornous"] = True
Console.error('Invalid return type "%s" in %s' % (returnTypeEntry["name"], className))
if not "pseudo" in returnTypeEntry and returnTypeEntry["name"] in knownClasses:
returnTypeEntry["linkable"] = True
elif not item["type"] in builtinTypes and not item["type"] in pseudoTypes and not item["type"] in additionalTypes:
item["errornous"] = True
Console.error('Invalid type "%s" in %s' % (item["type"], className))
# Process doc
if "doc" in item:
def processInternalLink(match):
linkUrl = match.group(2)
if linkUrl.startswith("#"):
linkCheck = checkInternalLink(linkUrl[1:], className)
if linkCheck is not True:
item["errornous"] = True
if sectionName:
Console.error("%s in %s:%s~%s" % (linkCheck, sectionName, className, name))
else:
Console.error("%s in %s" % (linkCheck, className))
linkExtract.sub(processInternalLink, item["doc"])
def inspect(*param, **args):
if args["severity"] > 20:
Console.error("Critical error occoured:")
Console.error(param[0])
def __init__(self, filename):
try:
self.fp = open(filename, "rb")
except IOError as err:
Console.error("Could not open file: %s" % filename)
raise err
def update(url, version, path, update=True, submodules=True):
"""Clones the given repository URL (optionally with overriding/update features)"""
# Prepend git+ so that user knows that we identified the URL as git repository
if not url.startswith("git+"):
url = "git+%s" % url
old = os.getcwd()
if os.path.exists(path) and os.path.exists(os.path.join(path, ".git")):
if not os.path.exists(os.path.join(path, ".git", "HEAD")):
Console.error("Invalid git clone. Cleaning up...")
shutil.rmtree(path)
else:
os.chdir(path)
revision = executeCommand(["git", "rev-parse", "HEAD"], "Could not detect current revision")
if update and (version == "master" or "refs/heads/" in version):
if update:
Console.info("Updating %s", Console.colorize("%s @ " % url, "bold") + Console.colorize(version, "magenta"))
Console.indent()
try:
executeCommand(["git", "fetch", "-q", "--depth", "1", "origin", version], "Could not fetch updated revision!")
executeCommand(["git", "reset", "-q", "--hard", "FETCH_HEAD"], "Could not update checkout!")
newRevision = executeCommand(["git", "rev-parse", "HEAD"], "Could not detect current revision")
if revision != newRevision:
Console.info("Updated from %s to %s", revision[:10], newRevision[:10])
revision = newRevision
if submodules and os.path.exists(".gitmodules"):
Console.info("Updating sub modules (this might take some time)...")
executeCommand("git submodule update --recursive", "Could not initialize sub modules")
except Exception:
Console.error("Error during git transaction! Could not update clone.")
Console.error("Please verify that the host is reachable or disable automatic branch updates.")
Console.outdent()
os.chdir(old)
return
except KeyboardInterrupt:
print()
Console.error("Git transaction was aborted by user!")
Console.outdent()
os.chdir(old)
return
Console.outdent()
else:
Console.debug("Updates disabled")
else:
Console.debug("Using existing clone")
os.chdir(old)
return revision
Console.info("Cloning %s", Console.colorize("%s @ " % url, "bold") + Console.colorize(version, "magenta"))
Console.indent()
os.makedirs(path)
os.chdir(path)
try:
# cut of "git+" prefix
remoteurl = url[4:]
executeCommand(["git", "init", "."], "Could not initialize GIT repository!")
executeCommand(["git", "remote", "add", "origin", remoteurl], "Could not register remote repository!")
executeCommand(["git", "fetch", "-q", "--depth", "1", "origin", version], "Could not fetch revision!")
executeCommand(["git", "reset", "-q", "--hard", "FETCH_HEAD"], "Could not update checkout!")
revision = executeCommand(["git", "rev-parse", "HEAD"], "Could not detect current revision")
if submodules and os.path.exists(".gitmodules"):
Console.info("Updating sub modules (this might take some time)...")
executeCommand("git submodule update --init --recursive", "Could not initialize sub modules")
except Exception:
Console.error("Error during git transaction! Intitial clone required for continuing!")
Console.error("Please verify that the host is reachable.")
Console.error("Cleaning up...")
os.chdir(old)
shutil.rmtree(path)
Console.outdent()
return
except KeyboardInterrupt:
print()
Console.error("Git transaction was aborted by user!")
Console.error("Cleaning up...")
os.chdir(old)
shutil.rmtree(path)
Console.outdent()
return
os.chdir(old)
Console.outdent()
return revision
def skip(self):
"""Eats comments and whitespace."""
input = self.source
startLine = self.line
# Whether this is the first called as happen on start parsing a file (eat leading comments/white space)
startOfFile = self.cursor is 0
indent = ""
while (True):
if len(input) > self.cursor:
ch = input[self.cursor]
else:
return
self.cursor += 1
if len(input) > self.cursor:
next = input[self.cursor]
else:
next = None
if ch == "\n" and not self.scanNewlines:
self.line += 1
indent = ""
elif ch == "/" and next == "*":
self.cursor += 1
text = "/*"
inline = startLine == self.line and startLine > 1
commentStartLine = self.line
if startLine == self.line and not startOfFile:
mode = "inline"
elif (self.line-1) > startLine:
# distance before this comment means it is a comment block for a whole section (multiple lines of code)
mode = "section"
else:
# comment for maybe multiple following lines of code, but not that important (no visual white space divider)
mode = "block"
while (True):
try:
ch = input[self.cursor]
self.cursor += 1
except IndexError:
raise ParseError("Unterminated comment", self.fileId, self.line)
if ch == "*":
next = input[self.cursor]
if next == "/":
text += "*/"
self.cursor += 1
break
elif ch == "\n":
self.line += 1
text += ch
# Filter escaping on slash-star combinations in comment text
text = text.replace("*\/", "*/")
try:
self.comments.append(Comment.Comment(text, mode, commentStartLine, indent, self.fileId))
except Comment.CommentException as commentError:
Console.error("Ignoring comment in %s: %s", self.fileId, commentError)
elif ch == "/" and next == "/":
self.cursor += 1
text = "//"
if startLine == self.line and not startOfFile:
mode = "inline"
elif (self.line-1) > startLine:
# distance before this comment means it is a comment block for a whole section (multiple lines of code)
mode = "section"
else:
# comment for maybe multiple following lines of code, but not that important (no visual white space divider)
mode = "block"
while (True):
try:
ch = input[self.cursor]
self.cursor += 1
except IndexError:
# end of file etc.
break
if ch == "\n":
self.line += 1
break
text += ch
try:
self.comments.append(Comment.Comment(text, mode, self.line-1, "", self.fileId))
except Comment.CommentException:
Console.error("Ignoring comment in %s: %s", self.fileId, commentError)
# check for whitespace, also for special cases like 0xA0
elif ch in "\xA0 \t":
indent += ch
else:
self.cursor -= 1
return
def write(self, distFolder, classFilter=None, callback="apiload", showInternals=False, showPrivates=False, printErrors=True, highlightCode=True):
"""
Writes API data generated from JavaScript into distFolder
:param distFolder: Where to store the API data
:param classFilter: Tuple of classes or method to use for filtering
:param callback: Name of callback to use for loading or None if pure JSON should be used
:param showInternals: Include internal methods inside API data
:param showPrivates: Include private methods inside API data
:param printErrors: Whether errors should be printed to the console
:param highlightCode: Whether to enable code highlighting using Pygments
:type distFolder: str
:type classFilter: tuple or function
:type callback: function
:type showInternals: bool
:type showPrivates: bool
:type printErrors: bool
:type highlightCode: bool
"""
#
# Collecting
#
Console.info("Collecting API Data...")
Console.indent()
apiData = {}
highlightedCode = {}
for project in self.__session.getProjects():
classes = project.getClasses()
Console.info("Loading API of project %s: %s...", Console.colorize(project.getName(), "bold"), Console.colorize("%s classes" % len(classes), "cyan"))
Console.indent()
for className in classes:
if self.__isIncluded(className, classFilter):
data = classes[className].getApi(highlightCode)
if not data.isEmpty:
apiData[className] = data
highlightedCode[className] = classes[className].getHighlightedCode()
else:
Console.info("Skipping %s, class is empty." % className)
Console.outdent()
Console.outdent()
#
# Processing
#
Console.info("Processing API Data...")
Console.indent()
data, index, search = self.__process(apiData, classFilter=classFilter, internals=showInternals, privates=showPrivates, printErrors=printErrors, highlightCode=highlightCode)
Console.outdent()
#
# Writing
#
Console.info("Storing API data...")
Console.indent()
writeCounter = 0
extension = "js" if callback else "json"
compress = True
class JsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, set):
return list(obj)
return json.JSONEncoder.default(self, obj)
def encode(content, name):
if compress:
jsonContent = json.dumps(content, sort_keys=True, cls=JsonEncoder, separators=(',',':'))
else:
jsonContent = json.dumps(content, sort_keys=True, cls=JsonEncoder, indent=2)
if callback:
return "%s(%s,'%s');" % (callback, jsonContent, name)
else:
return jsonContent
Console.info("Saving class data (%s files)...", len(data))
Console.indent()
for className in data:
try:
classData = data[className]
if type(classData) is dict:
classExport = classData
else:
classExport = classData.export()
File.write(self.__session.expandFileName(os.path.join(distFolder, "%s.%s" % (className, extension))), encode(classExport, className))
except TypeError as writeError:
Console.error("Could not write API data of: %s: %s", className, writeError)
continue
Console.outdent()
if highlightCode:
Console.info("Saving highlighted code (%s files)...", len(highlightedCode))
Console.indent()
for className in highlightedCode:
try:
File.write(self.__session.expandFileName(os.path.join(distFolder, "%s.html" % className)), highlightedCode[className])
except TypeError as writeError:
Console.error("Could not write highlighted code of: %s: %s", className, writeError)
continue
Console.outdent()
Console.info("Writing index...")
Console.indent()
File.write(self.__session.expandFileName(os.path.join(distFolder, "meta-index.%s" % extension)), encode(index, "meta-index"))
File.write(self.__session.expandFileName(os.path.join(distFolder, "meta-search.%s" % extension)), encode(search, "meta-search"))
Console.outdent()
Console.outdent()
def skip(self):
"""Eats comments and whitespace."""
input = self.source
startLine = self.line
# Whether this is the first called as happen on start parsing a file (eat leading comments/white space)
startOfFile = self.cursor is 0
indent = ""
while (True):
if len(input) > self.cursor:
ch = input[self.cursor]
else:
return
self.cursor += 1
if len(input) > self.cursor:
next = input[self.cursor]
else:
next = None
if ch == "\n" and not self.scanNewlines:
self.line += 1
indent = ""
elif ch == "/" and next == "*":
self.cursor += 1
text = "/*"
inline = startLine == self.line and startLine > 1
commentStartLine = self.line
if startLine == self.line and not startOfFile:
mode = "inline"
elif (self.line - 1) > startLine:
# distance before this comment means it is a comment block for a whole section (multiple lines of code)
mode = "section"
else:
# comment for maybe multiple following lines of code, but not that important (no visual white space divider)
mode = "block"
while (True):
try:
ch = input[self.cursor]
self.cursor += 1
except IndexError:
raise ParseError("Unterminated comment", self.fileId,
self.line)
if ch == "*":
next = input[self.cursor]
if next == "/":
text += "*/"
self.cursor += 1
break
elif ch == "\n":
self.line += 1
text += ch
# Filter escaping on slash-star combinations in comment text
text = text.replace("*\/", "*/")
try:
self.comments.append(
Comment.Comment(text, mode, commentStartLine, indent,
self.fileId))
except Comment.CommentException as commentError:
Console.error("Ignoring comment in %s: %s", self.fileId,
commentError)
elif ch == "/" and next == "/":
self.cursor += 1
text = "//"
if startLine == self.line and not startOfFile:
mode = "inline"
elif (self.line - 1) > startLine:
# distance before this comment means it is a comment block for a whole section (multiple lines of code)
mode = "section"
else:
# comment for maybe multiple following lines of code, but not that important (no visual white space divider)
mode = "block"
while (True):
try:
ch = input[self.cursor]
self.cursor += 1
except IndexError:
# end of file etc.
break
if ch == "\n":
self.line += 1
break
text += ch
try:
self.comments.append(
Comment.Comment(text, mode, self.line - 1, "",
self.fileId))
except Comment.CommentException:
Console.error("Ignoring comment in %s: %s", self.fileId,
commentError)
# check for whitespace, also for special cases like 0xA0
elif ch in "\xA0 \t":
indent += ch
else:
self.cursor -= 1
return
def __init__(self, filename):
try:
self.fp = open(filename, "rb")
except IOError as err:
Console.error("Could not open file: %s" % filename)
raise err
def scan(self):
if self.scanned:
return
updatemsg = "[updated]" if self.__modified else "[cached]"
Console.info("Scanning project %s %s...", self.__name,
Console.colorize(updatemsg, "grey"))
Console.indent()
# Support for pre-initialize projects...
setup = self.__setup
if setup and self.__modified:
Console.info("Running setup...")
Console.indent()
for cmd in setup:
Console.info("Executing %s...", cmd)
result = None
try:
result = None
result = Util.executeCommand(
cmd,
"Failed to execute setup command %s" % cmd,
path=self.__path)
except Exception as ex:
if result:
Console.error(result)
raise UserError("Could not scan project %s: %s" %
(self.__name, ex))
Console.outdent()
# Processing custom content section. Only supports classes and assets.
if self.__config.has("content"):
self.kind = "manual"
self.__addContent(self.__config.get("content"))
# Application projects
elif self.__hasDir("source"):
self.kind = "application"
if self.__hasDir("source/class"):
self.__addDir("source/class", "classes")
if self.__hasDir("source/asset"):
self.__addDir("source/asset", "assets")
if self.__hasDir("source/translation"):
self.__addDir("source/translation", "translations")
# Compat - please change to class/style/asset instead
elif self.__hasDir("src"):
self.kind = "resource"
self.__addDir("src", "classes")
# Resource projects
else:
self.kind = "resource"
if self.__hasDir("class"):
self.__addDir("class", "classes")
if self.__hasDir("asset"):
self.__addDir("asset", "assets")
if self.__hasDir("translation"):
self.__addDir("translation", "translations")
# Generate summary
summary = []
for section in ["classes", "assets", "translations"]:
content = getattr(self, section, None)
if content:
summary.append("%s %s" % (len(content), section))
# Print out
if summary:
Console.info("Done %s: %s" %
(Console.colorize("[%s]" % self.kind, "grey"),
Console.colorize(", ".join(summary), "green")))
else:
Console.error("Project is empty!")
self.scanned = True
Console.outdent()
def requestUrl(url,
content_type="text/plain",
headers=None,
method="GET",
port=None,
body="",
user=None,
password=None):
"""Generic HTTP request wrapper with support for basic authentification and automatic parsing of response content"""
Console.info("Opening %s request to %s..." % (method, url))
parsed = urllib.parse.urlparse(url)
if parsed.scheme == "http":
request = http.client.HTTPConnection(parsed.netloc)
elif parsed.scheme == "https":
request = http.client.HTTPSConnection(parsed.netloc)
else:
raise Exception("Unsupported url: %s" % url)
if parsed.query:
request.putrequest(method, parsed.path + "?" + parsed.query)
else:
request.putrequest(method, parsed.path)
request.putheader("Content-Type", content_type)
request.putheader("Content-Length", str(len(body)))
if user is not None and password is not None:
auth = "Basic %s" % base64.b64encode(
("%s:%s" % (user, password)).encode("utf-8")).decode("utf-8")
request.putheader("Authorization", auth)
request.endheaders()
if body:
Console.info("Sending data (%s bytes)..." % len(body))
else:
Console.info("Sending request...")
Console.indent()
request.send(body)
response = request.getresponse()
res_code = int(response.getcode())
res_headers = dict(response.getheaders())
res_content = response.read()
res_success = False
if res_code >= 200 and res_code <= 300:
Console.debug("HTTP Success!")
res_success = True
else:
Console.error("HTTP Failure Code: %s!", res_code)
if "Content-Type" in res_headers:
res_type = res_headers["Content-Type"]
if ";" in res_type:
res_type = res_type.split(";")[0]
if res_type in ("application/json", "text/html", "text/plain"):
res_content = res_content.decode("utf-8")
if res_type == "application/json":
res_content = json.loads(res_content)
if "error" in res_content:
Console.error("Error %s: %s", res_content["error"],
res_content["reason"])
elif "reason" in res_content:
Console.info("Success: %s" % res_content["reason"])
Console.outdent()
return res_success, res_headers, res_content
def scan(self):
if self.scanned:
return
updatemsg = "[updated]" if self.__modified else "[cached]"
if self.version:
Console.info("Scanning %s @ %s %s...",
Console.colorize(self.getName(), "bold"),
Console.colorize(self.version, "magenta"),
Console.colorize(updatemsg, "grey"))
else:
Console.info("Scanning %s %s...",
Console.colorize(self.getName(), "bold"),
Console.colorize(updatemsg, "grey"))
Console.indent()
# Support for pre-initialize projects...
setup = self.__setup
if setup and self.__modified:
Console.info("Running setup...")
Console.indent()
for cmd in setup:
Console.info("Executing %s...", cmd)
result = None
try:
result = None
result = Util.executeCommand(
cmd,
"Failed to execute setup command %s" % cmd,
path=self.__path)
except Exception as ex:
if result:
Console.error(result)
raise UserError("Could not scan project %s: %s" %
(self.__name, ex))
Console.outdent()
# Processing custom content section. Only supports classes and assets.
if self.__config.has("content"):
self.kind = "manual"
self.__addContent(self.__config.get("content"))
else:
# Read scan path from config
if not self.__config.has("scan"):
if self.__hasDir("source"):
self.kind = "application"
scan = self.__resolveScanConfig(structures[self.kind])
elif self.__hasDir("src"):
self.kind = "resource"
scan = self.__resolveScanConfig(structures[self.kind])
else:
self.kind = "flat"
scan = self.__resolveScanConfig(structures[self.kind])
else:
scan = self.__resolveScanConfig(self.__config.get("scan"))
for config in scan:
if isinstance(config["paths"], str):
self.__addDir(config["paths"], config["regex"],
config["type"], config["package"])
else:
for path in config["paths"]:
self.__addDir(path, config["regex"], config["type"],
config["package"])
# Generate summary
summary = []
for section in self.items.keys():
content = self.items[section]
name, constructor = self.__resolveConstructor(section)
if content:
summary.append(
Console.colorize("%s %s" % (len(content), name),
"magenta"))
# Print out
if summary:
Console.info("Content: %s" % (", ".join(summary)))
self.scanned = True
Console.outdent()
def write(self,
distFolder,
classFilter=None,
callback="apiload",
showInternals=False,
showPrivates=False,
printErrors=True,
highlightCode=True):
"""
Writes API data generated from JavaScript into distFolder.
:param distFolder: Where to store the API data
:param classFilter: Tuple of classes or method to use for filtering
:param callback: Name of callback to use for loading or None if pure JSON should be used
:param showInternals: Include internal methods inside API data
:param showPrivates: Include private methods inside API data
:param printErrors: Whether errors should be printed to the console
:param highlightCode: Whether to enable code highlighting using Pygments
:type distFolder: str
:type classFilter: tuple or function
:type callback: function
:type showInternals: bool
:type showPrivates: bool
:type printErrors: bool
:type highlightCode: bool
"""
#
# Collecting
#
Console.info("Collecting API Data...")
Console.indent()
apiData = {}
highlightedCode = {}
for project in self.__session.getProjects():
classes = project.getScripts()
Console.info("Loading API of project %s: %s...",
Console.colorize(project.getName(), "bold"),
Console.colorize("%s classes" % len(classes), "cyan"))
Console.indent()
for className in classes:
if self.__isIncluded(className, classFilter):
data = classes[className].getApi(highlightCode)
if not data.isEmpty:
apiData[className] = data
highlightedCode[className] = classes[
className].getHighlightedCode()
else:
Console.info("Skipping %s, class is empty." %
className)
Console.outdent()
Console.outdent()
#
# Processing
#
Console.info("Processing API Data...")
Console.indent()
data, index, search = self.__process(apiData,
classFilter=classFilter,
internals=showInternals,
privates=showPrivates,
printErrors=printErrors,
highlightCode=highlightCode)
Console.outdent()
#
# Writing
#
Console.info("Storing API data...")
Console.indent()
writeCounter = 0
extension = "js" if callback else "json"
compress = True
class JsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, set):
return list(obj)
return json.JSONEncoder.default(self, obj)
def encode(content, name):
if compress:
jsonContent = json.dumps(content,
sort_keys=True,
cls=JsonEncoder,
separators=(',', ':'))
else:
jsonContent = json.dumps(content,
sort_keys=True,
cls=JsonEncoder,
indent=2)
if callback:
return "%s(%s,'%s');" % (callback, jsonContent, name)
else:
return jsonContent
Console.info("Saving class data (%s files)...", len(data))
Console.indent()
for className in data:
try:
classData = data[className]
if isinstance(classData, dict):
classExport = classData
else:
classExport = classData.export()
File.write(
self.__profile.expandFileName(
os.path.join(distFolder,
"%s.%s" % (className, extension))),
encode(classExport, className))
except TypeError as writeError:
Console.error("Could not write API data of: %s: %s", className,
writeError)
continue
Console.outdent()
if highlightCode:
Console.info("Saving highlighted code (%s files)...",
len(highlightedCode))
Console.indent()
for className in highlightedCode:
try:
File.write(
self.__profile.expandFileName(
os.path.join(distFolder, "%s.html" % className)),
highlightedCode[className])
except TypeError as writeError:
Console.error(
"Could not write highlighted code of: %s: %s",
className, writeError)
continue
Console.outdent()
Console.info("Writing index...")
Console.indent()
File.write(
self.__profile.expandFileName(
os.path.join(distFolder, "meta-index.%s" % extension)),
encode(index, "meta-index"))
File.write(
self.__profile.expandFileName(
os.path.join(distFolder, "meta-search.%s" % extension)),
encode(search, "meta-search"))
Console.outdent()
Console.outdent()
def inspect(*param, **args):
if args["severity"] > 20:
Console.error("Critical error occoured:")
Console.error(param[0])
def scan(self):
if self.scanned:
return
updatemsg = "[updated]" if self.__modified else "[cached]"
Console.info("Scanning project %s %s...", self.__name, Console.colorize(updatemsg, "grey"))
Console.indent()
# Support for pre-initialize projects...
setup = self.__setup
if setup and self.__modified:
Console.info("Running setup...")
Console.indent()
for cmd in setup:
Console.info("Executing %s...", cmd)
result = None
try:
result = None
result = Util.executeCommand(cmd, "Failed to execute setup command %s" % cmd, path=self.__path)
except Exception as ex:
if result:
Console.error(result)
raise UserError("Could not scan project %s: %s" % (self.__name, ex))
Console.outdent()
# Processing custom content section. Only supports classes and assets.
if self.__config.has("content"):
self.kind = "manual"
self.__addContent(self.__config.get("content"))
# Application projects
elif self.__hasDir("source"):
self.kind = "application"
if self.__hasDir("source/class"):
self.__addDir("source/class", "classes")
if self.__hasDir("source/style"):
self.__addDir("source/style", "styles")
if self.__hasDir("source/asset"):
self.__addDir("source/asset", "assets")
if self.__hasDir("source/translation"):
self.__addDir("source/translation", "translations")
# Compat - please change to class/style/asset instead
elif self.__hasDir("src"):
self.kind = "resource"
self.__addDir("src", "classes")
# Resource projects
else:
self.kind = "resource"
if self.__hasDir("class"):
self.__addDir("class", "classes")
if self.__hasDir("style"):
self.__addDir("style", "styles")
if self.__hasDir("asset"):
self.__addDir("asset", "assets")
if self.__hasDir("translation"):
self.__addDir("translation", "translations")
# Generate summary
summary = []
for section in ["classes", "styles", "translations", "assets"]:
content = getattr(self, section, None)
if content:
summary.append("%s %s" % (len(content), section))
# Print out
if summary:
Console.info("Content: %s" % (Console.colorize(", ".join(summary), "green")))
else:
Console.error("Project is empty!")
self.scanned = True
Console.outdent()
def checkLinksInItem(item):
# Process types
if "type" in item:
if item["type"] == "Function":
# Check param types
if "params" in item:
for paramName in item["params"]:
paramEntry = item["params"][paramName]
if "type" in paramEntry:
for paramTypeEntry in paramEntry["type"]:
if not paramTypeEntry[
"name"] in knownClasses and not paramTypeEntry[
"name"] in additionalTypes and not (
"builtin" in paramTypeEntry
or "pseudo"
in paramTypeEntry):
item["errornous"] = True
Console.error(
'Invalid param type "%s" in %s' %
(paramTypeEntry["name"],
className))
if not "pseudo" in paramTypeEntry and paramTypeEntry[
"name"] in knownClasses:
paramTypeEntry["linkable"] = True
# Check return types
if "returns" in item:
for returnTypeEntry in item["returns"]:
if not returnTypeEntry[
"name"] in knownClasses and not returnTypeEntry[
"name"] in additionalTypes and not (
"builtin" in returnTypeEntry
or "pseudo" in returnTypeEntry):
item["errornous"] = True
Console.error(
'Invalid return type "%s" in %s' %
(returnTypeEntry["name"], className))
if not "pseudo" in returnTypeEntry and returnTypeEntry[
"name"] in knownClasses:
returnTypeEntry["linkable"] = True
elif not item["type"] in knownClasses and not item[
"type"] in builtinTypes and not item[
"type"] in pseudoTypes and not item[
"type"] in additionalTypes:
item["errornous"] = True
Console.error('Invalid type "%s" in %s' %
(item["type"], className))
# Process doc
if "doc" in item:
def processInternalLink(match):
linkUrl = match.group(2)
if linkUrl.startswith("#"):
linkCheck = checkInternalLink(linkUrl[1:], className)
if linkCheck is not True:
item["errornous"] = True
if sectionName:
Console.error(
"%s in %s:%s~%s" %
(linkCheck, sectionName, className, name))
else:
Console.error("%s in %s" %
(linkCheck, className))
linkExtract.sub(processInternalLink, item["doc"])
def process(url, start=0, fetch=50):
""" Main processing engine """
pos = start
# End will be updated during each request with incoming data
end = pos + fetch
Console.header("Tumblr Import")
Console.info("Importing data...")
Console.indent()
while pos < end:
Console.info("Requesting %s-%s of %s" % (pos, pos+fetch-1, end))
response = requests.get(url % (pos, fetch))
if response.status_code != 200:
raise Exception("Error during communication with Tumblr: %s" % r.status)
tree = ElementTree.fromstring(response.content)
# This element contains all posts
allPosts = tree.find("posts")
# Update end pointer
end = int(allPosts.get("total"))
# Iterate trough all posts
for post in allPosts:
postType = post.get("type")
postTimeStamp = post.get("unix-timestamp")
postExportDate = str(datetime.datetime.fromtimestamp(int(postTimeStamp)))
postSlug = post.get("slug")
postFormat = post.get("format")
postDateOnly = postExportDate[0:postExportDate.find(" ")]
postFileName = "%s-%s" % (postDateOnly, postSlug)
if postType == "quote":
quoteText = post.find("quote-text").text
quoteComment = post.find("quote-source").text
# Post-process
quoteText = markdownify.markdownify("<blockquote>" + quoteText + "</blockquote>").rstrip("\n").lstrip("\n")
quoteComment = markdownify.markdownify(quoteComment).rstrip("\n")
fileContent = quoteTemplate % (postSlug, postExportDate, quoteText + "\n\n" + quoteComment)
elif postType == "photo":
photoText = post.find("photo-caption").text
try:
photoLinkUrl = post.find("photo-link-url").text
except:
photoLinkUrl = None
photoUrl = post.find("photo-url").text
# Post-process
photoText = markdownify.markdownify(photoText).rstrip("\n")
# Downloading image
photoResponse = requests.get(photoUrl, allow_redirects=True)
if photoResponse.status_code != 200:
Console.error("Unable to load photo. Status: %s; URL: %s", photoResponse.status_code, photoUrl)
continue
# Build extension based on response headers (safer than using file extension)
photoType = photoResponse.headers["content-type"]
if "png" in photoType:
photoExtension = ".png"
elif "jpeg" in photoType or "jpg" in photoType:
photoExtension = ".jpeg"
elif "gif" in photoType:
photoExtension = ".gif"
else:
Console.error("Unknown photo format: %s; Status: %s; URL: %s", photoType, photoResponse.status_code, photoUrl)
continue
# Generating checksum
photoHash = hashlib.sha1(photoResponse.content).hexdigest()
# Generate file name and path from existing data
photoFileName = "%s-%s-%s%s" % (postDateOnly, postSlug, photoHash[0:10], photoExtension)
photoPath = os.path.join(photoFolder, photoFileName)
# Do not repeatly write identical files
if not os.path.exists(photoPath):
photoFile = open(photoPath, "wb")
photoFile.write(photoResponse.content)
photoFile.close()
# Generate basic image tag
photoAsset = '<img src="{{@asset.url %s/%s/%s}}" alt=""/>' % (projectName, photoAssetFolder, photoFileName)
# Wrap with a link when it should be link to an external site
if photoLinkUrl:
photoAsset = '<a href="%s">%s</a>' % (photoLinkUrl, photoAsset)
fileContent = photoTemplate % (postSlug, postExportDate, photoAsset + "\n\n" + photoText)
elif postType == "link":
linkUrl = post.find("link-url").text
try:
linkText = post.find("link-text").text
except:
linkText = linkUrl
# Post-process
if linkText != linkUrl:
linkText = markdownify.markdownify(linkText).rstrip("\n")
fileContent = linkTemplate % (postSlug, postExportDate, "[%s](%s)" % (linkText, linkUrl))
elif postType == "video":
videoCode = post.find("video-source").text
videoText = post.find("video-caption").text
# Post-process
videoText = markdownify.markdownify(videoText).rstrip("\n")
fileContent = videoTemplate % (postSlug, postExportDate, videoCode + "\n\n" + videoText)
elif postType == "regular":
postText = post.find("regular-body").text
try:
postTitle = post.find("regular-title").text
except:
# Ignore posts without title
Console.warn("Ignoring post without title!")
continue
postText = markdownify.markdownify(postText).rstrip("\n")
fileContent = regularTemplate % (postSlug, postExportDate, postTitle, postText)
else:
Console.warn("Unknown POST-TYPE: %s" % postType)
print(ElementTree.dump(post))
continue
# Write post file
fileHandle = open(os.path.join(postFolder, postDateOnly + "-" + postType + "-" + postSlug + ".markdown"), "w")
fileHandle.write(fileContent)
fileHandle.close()
# Update for next requests
pos = pos + fetch
Console.outdent()
Console.info("Successfully imported")
def update(url, version, path, update=True, submodules=True):
"""Clones the given repository URL (optionally with overriding/update features)"""
# Prepend git+ so that user knows that we identified the URL as git repository
if not url.startswith("git+"):
url = "git+%s" % url
old = os.getcwd()
if os.path.exists(path) and os.path.exists(os.path.join(path, ".git")):
if not os.path.exists(os.path.join(path, ".git", "HEAD")):
Console.error("Invalid git clone. Cleaning up...")
shutil.rmtree(path)
else:
os.chdir(path)
revision = executeCommand(["git", "rev-parse", "HEAD"], "Could not detect current revision")
if update and (version == "master" or "refs/heads/" in version):
if update:
Console.info("Updating %s", Console.colorize("%s @ " % url, "bold") + Console.colorize(version, "magenta"))
Console.indent()
try:
executeCommand(["git", "fetch", "-q", "--depth", "1", "origin", version], "Could not fetch updated revision!")
executeCommand(["git", "reset", "-q", "--hard", "FETCH_HEAD"], "Could not update checkout!")
newRevision = executeCommand(["git", "rev-parse", "HEAD"], "Could not detect current revision")
if revision != newRevision:
Console.info("Updated from %s to %s", revision[:10], newRevision[:10])
revision = newRevision
if submodules and os.path.exists(".gitmodules"):
Console.info("Updating sub modules (this might take some time)...")
executeCommand("git submodule update --recursive", "Could not initialize sub modules")
except Exception:
Console.error("Error during git transaction! Could not update clone.")
Console.error("Please verify that the host is reachable or disable automatic branch updates.")
Console.outdent()
os.chdir(old)
return
except KeyboardInterrupt:
print()
Console.error("Git transaction was aborted by user!")
Console.outdent()
os.chdir(old)
return
Console.outdent()
else:
Console.debug("Updates disabled")
else:
Console.debug("Using existing clone")
os.chdir(old)
return revision
Console.info("Cloning %s", Console.colorize("%s @ " % url, "bold") + Console.colorize(version, "magenta"))
Console.indent()
os.makedirs(path)
os.chdir(path)
try:
# cut of "git+" prefix
remoteurl = url[4:]
executeCommand(["git", "init", "."], "Could not initialize GIT repository!")
executeCommand(["git", "remote", "add", "origin", remoteurl], "Could not register remote repository!")
executeCommand(["git", "fetch", "-q", "--depth", "1", "origin", version], "Could not fetch revision!")
executeCommand(["git", "reset", "-q", "--hard", "FETCH_HEAD"], "Could not update checkout!")
revision = executeCommand(["git", "rev-parse", "HEAD"], "Could not detect current revision")
if submodules and os.path.exists(".gitmodules"):
Console.info("Updating sub modules (this might take some time)...")
executeCommand("git submodule update --init --recursive", "Could not initialize sub modules")
except Exception:
Console.error("Error during git transaction! Intitial clone required for continuing!")
Console.error("Please verify that the host is reachable.")
Console.error("Cleaning up...")
os.chdir(old)
shutil.rmtree(path)
Console.outdent()
return
except KeyboardInterrupt:
print()
Console.error("Git transaction was aborted by user!")
Console.error("Cleaning up...")
os.chdir(old)
shutil.rmtree(path)
Console.outdent()
return
os.chdir(old)
Console.outdent()
return revision
#
# Jasy - Web Tooling Framework
# Copyright 2010-2012 Zynga Inc.
#
import jasy.core.Console as Console
from distutils.version import StrictVersion, LooseVersion
try:
import pip
except ImportError:
Console.error("pip is required to run JASY!")
sys.exit(1)
needs = [
{
"packageName": "Pygments",
"minVersion": "1.5",
"installPath": "'pip3 install Pygments'",
"updatePath": "'pip3 install --upgrade pygments'"
},
{
"packageName": "polib",
"minVersion": "1.0.2",
"installPath": "'pip3 install polib'",
"updatePath": "'pip3 install --upgrade polib'"
},
{
"packageName": "requests",
"minVersion": "1.1",
def scan(self):
if self.scanned:
return
updatemsg = "[updated]" if self.__modified else "[cached]"
if self.version:
Console.info("Scanning %s @ %s %s...", Console.colorize(self.getName(), "bold"), Console.colorize(self.version, "magenta"), Console.colorize(updatemsg, "grey"))
else:
Console.info("Scanning %s %s...", Console.colorize(self.getName(), "bold"), Console.colorize(updatemsg, "grey"))
Console.indent()
# Support for pre-initialize projects...
setup = self.__setup
if setup and self.__modified:
Console.info("Running setup...")
Console.indent()
for cmd in setup:
Console.info("Executing %s...", cmd)
result = None
try:
result = None
result = Util.executeCommand(cmd, "Failed to execute setup command %s" % cmd, path=self.__path)
except Exception as ex:
if result:
Console.error(result)
raise UserError("Could not scan project %s: %s" % (self.__name, ex))
Console.outdent()
# Processing custom content section. Only supports classes and assets.
if self.__config.has("content"):
self.kind = "manual"
self.__addContent(self.__config.get("content"))
else:
# Read scan path from config
if not self.__config.has("scan"):
if self.__hasDir("source"):
self.kind = "application"
scan = self.__resolveScanConfig(structures[self.kind])
elif self.__hasDir("src"):
self.kind = "resource"
scan = self.__resolveScanConfig(structures[self.kind])
else:
self.kind = "flat"
scan = self.__resolveScanConfig(structures[self.kind])
else:
scan = self.__resolveScanConfig(self.__config.get("scan"))
for config in scan:
if isinstance(config["paths"], str):
self.__addDir(config["paths"], config["regex"], config["type"], config["package"])
else:
for path in config["paths"]:
self.__addDir(path, config["regex"], config["type"], config["package"])
# Generate summary
summary = []
for section in self.items.keys():
content = self.items[section]
name, constructor = self.__resolveConstructor(section)
if content:
summary.append(Console.colorize("%s %s" % (len(content), name), "magenta"))
# Print out
if summary:
Console.info("Content: %s" % (", ".join(summary)))
self.scanned = True
Console.outdent()
def requestUrl(url, content_type="text/plain", headers=None, method="GET", port=None, body="", user=None, password=None):
"""Generic HTTP request wrapper with support for basic authentification and automatic parsing of response content"""
Console.info("Opening %s request to %s..." % (method, url))
parsed = urllib.parse.urlparse(url)
if parsed.scheme== "http":
request = http.client.HTTPConnection(parsed.netloc)
elif parsed.scheme== "https":
request = http.client.HTTPSConnection(parsed.netloc)
else:
raise Exception("Unsupported url: %s" % url)
if parsed.query:
request.putrequest(method, parsed.path + "?" + parsed.query)
else:
request.putrequest(method, parsed.path)
request.putheader("Content-Type", content_type)
request.putheader("Content-Length", str(len(body)))
if user is not None and password is not None:
auth = "Basic %s" % base64.b64encode(("%s:%s" % (user, password)).encode("utf-8")).decode("utf-8")
request.putheader("Authorization", auth)
request.endheaders()
if body:
Console.info("Sending data (%s bytes)..." % len(body))
else:
Console.info("Sending request...")
Console.indent()
request.send(body)
response = request.getresponse()
res_code = int(response.getcode())
res_headers = dict(response.getheaders())
res_content = response.read()
res_success = False
if res_code >= 200 and res_code <= 300:
Console.debug("HTTP Success!")
res_success = True
else:
Console.error("HTTP Failure Code: %s!", res_code)
if "Content-Type" in res_headers:
res_type = res_headers["Content-Type"]
if ";" in res_type:
res_type = res_type.split(";")[0]
if res_type in ("application/json", "text/html", "text/plain"):
res_content = res_content.decode("utf-8")
if res_type == "application/json":
res_content = json.loads(res_content)
if "error" in res_content:
Console.error("Error %s: %s", res_content["error"], res_content["reason"])
elif "reason" in res_content:
Console.info("Success: %s" % res_content["reason"])
Console.outdent()
return res_success, res_headers, res_content
#
# Jasy - Web Tooling Framework
# Copyright 2010-2012 Zynga Inc.
#
import jasy.core.Console as Console
from distutils.version import LooseVersion
try:
import pip
except ImportError:
Console.error("pip is required to run JASY!")
sys.exit(1)
needs = [{
"packageName": "Pygments",
"minVersion": "1.5",
"installPath": "'pip install Pygments'",
"updatePath": "'pip install --upgrade pygments'"
}, {
"packageName": "polib",
"minVersion": "1.0.1",
"installPath": "'pip install polib'",
"updatePath": "'pip install --upgrade polib'"
}, {
"packageName": "requests",
"minVersion": "0.14",
"installPath": "'pip install requests'",
"updatePath": "'pip install --upgrade requests'"
}, {
"packageName": "CherryPy",
