def generateCmake(buildType="Release",
cmakeToolchainFile="FrameworkInternals" + os.path.sep +
"default_configuration.cmake"):
"""Generates CMake header lists in various directories, and then calls cmake.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
transformDesignVerbose(
"AddressSpace" + os.path.sep +
"designToGeneratedCmakeAddressSpace.xslt",
"AddressSpace" + os.path.sep + "cmake_generated.cmake", 0, 0)
transformDesignVerbose(
"Device" + os.path.sep + "designToGeneratedCmakeDevice.xslt",
"Device" + os.path.sep + "generated" + os.path.sep +
"cmake_header.cmake", 0, 0)
print("Build type [" + buildType + "], Toolchain file [" +
cmakeToolchainFile + "]")
print("Calling CMake")
if platform.system() == "Windows":
subprocessWithImprovedErrors([
getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType,
"-DCMAKE_TOOLCHAIN_FILE=" + cmakeToolchainFile, "-G",
"Visual Studio 12 Win64", "."
], getCommand("cmake"))
elif platform.system() == "Linux":
subprocessWithImprovedErrors([
getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType,
"-DCMAKE_TOOLCHAIN_FILE=" + cmakeToolchainFile, "."
], getCommand("cmake"))
def generateCmake(buildType="Release",
cmakeToolchainFile="FrameworkInternals" + os.path.sep + "default_configuration.cmake"):
"""Generates CMake header lists in various directories, and then calls cmake.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
transformDesignVerbose("AddressSpace" + os.path.sep + "designToGeneratedCmakeAddressSpace.xslt",
"AddressSpace" + os.path.sep + "cmake_generated.cmake",
0, 0)
transformDesignVerbose("Device" + os.path.sep + "designToGeneratedCmakeDevice.xslt",
"Device" + os.path.sep + "generated" + os.path.sep + "cmake_header.cmake",
0, 0)
print("Build type ["+buildType+"], Toolchain file [" + cmakeToolchainFile + "]")
print("Calling CMake")
if platform.system() == "Windows":
subprocessWithImprovedErrors([getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType,
"-DCMAKE_TOOLCHAIN_FILE=" + cmakeToolchainFile,
"-G", "Visual Studio 12 Win64", "."],
getCommand("cmake"))
elif platform.system() == "Linux":
subprocessWithImprovedErrors([getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType,
"-DCMAKE_TOOLCHAIN_FILE=" + cmakeToolchainFile, "."],
getCommand("cmake"))
def automatedBuild(context, *args):
"""Method that generates the cmake headers, and after that calls make/vis-studio to compile your server.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
args2 = list(args) # we do it to be able to remove things via extract_argument, otherwise it is immutable tuple
(args2, builder) = extract_argument(args2, "--builder")
if builder is None: builder = BuilderDefault
if len(args2) > 1:
raise WrongArguments("Max one positional argument for build")
buildType = "Release" if len(args2) == 0 else args2[0]
if not buildType in ["Release","Debug"]:
raise Exception ("Only Release or Debug is accepted as the parameter. "
"If you are used to passing build config through here, note this version of quasar has separate command to do that: build_config")
generateCmake(context, *args)
print("Calling build, type [{0}], builder [{1}]".format(buildType, builder))
if platform.system() == "Windows":
try:
subprocessWithImprovedErrors( "cmake --build . --target ALL_BUILD --config "+buildType, 'visual studio build')
except Exception as e:
print("Build process error. Exception: [" + str(e) + "]")
elif platform.system() == "Linux":
if builder == BuilderDefault:
print('make -j$(nproc)')
process = subprocess.Popen(["nproc"], stdout=subprocess.PIPE)
out, err = process.communicate()
subprocessWithImprovedErrors([getCommand("make"), "-j" + str(int(out))], getCommand("make"))#the conversion from string to int and back to string is to remove all whitespaces and ensure that we have an integer
else:
subprocessWithImprovedErrors([getCommand("cmake"), "--build", ".", "--config", builder], "cmake --build")
def automatedBuild(projectSourceDir=None,
projectBinaryDir=None,
buildType="Release"):
"""Method that generates the cmake headers, and after that calls make/msbuild to compile your server.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
if not buildType in ["Release", "Debug"]:
raise Exception(
"Only Release or Debug is accepted as the parameter. "
"If you are used to passing build config through here, note this version of quasar has separate command to do that: build_config"
)
generateCmake(projectSourceDir, projectBinaryDir, buildType)
print('Calling make/msbuild')
if platform.system() == "Windows":
print('Calling visual studio vcvarsall to set the environment')
print(getCommand("vcvarsall") + ' amd64')
subprocessWithImprovedErrors(
"\"" + getCommand("vcvarsall") + '\" amd64',
"visual studio vcvarsall.bat")
print(
'msbuild ALL_BUILD.vcxproj /clp:ErrorsOnly /property:Platform=x64;Configuration='
+ buildType)
try:
subprocessWithImprovedErrors(
"\"" + getCommand("vcvarsall") + '\" amd64 && ' +
getCommand("msbuild") +
' ALL_BUILD.vcxproj /clp:ErrorsOnly /property:Platform=x64;Configuration='
+ buildType, "visual studio msbuild")
except Exception, e:
print("Build process error. Exception: [" + str(e) + "]")
def formatXml(inFileName, outFileName):
if platform.system() == "Windows":
subprocessWithImprovedErrorsPipeOutputToFile(
[getCommand("xmllint"), inFileName], outFileName,
getCommand("xmllint"))
elif platform.system() == "Linux":
subprocessWithImprovedErrorsPipeOutputToFile(
[getCommand("xmllint"), "--format", inFileName], outFileName,
getCommand("xmllint"))
def checkExecutableExists(executableKeyName, doesNotExistErrorMessage, executableArgument='-h'):
errorMessage = "executable [key:"+executableKeyName+", command: "+getCommand(executableKeyName)+"] cannot be found. Maybe it is not installed, or maybe it is not set in the PATH. \n"+doesNotExistErrorMessage
try:
returnCode = subprocess.call([getCommand(executableKeyName), executableArgument], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
if returnCode == 0:
printIfVerbose("executable [key:"+executableKeyName+", command: "+getCommand(executableKeyName)+"] exists")
else:
raise Exception(errorMessage)
except:
raise Exception(errorMessage)
def checkExecutableExists(executableKeyName, doesNotExistErrorMessage, executableArgument='-h'):
errorMessage = "executable [key:"+executableKeyName+", command: "+getCommand(executableKeyName)+"] cannot be found. Maybe it is not installed, or maybe it is not set in the PATH. \n"+doesNotExistErrorMessage
try:
returnCode = subprocess.call([getCommand(executableKeyName), executableArgument], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
if returnCode == 0:
printIfVerbose("executable [key:"+executableKeyName+", command: "+getCommand(executableKeyName)+"] exists")
else:
raise Exception(errorMessage)
except:
raise Exception(errorMessage)
def runDoxygen():
"""Runs doxygen in the documentation folder, making the tool generate documentation for the server automatically."""
baseDirectory = os.getcwd()
os.chdir(baseDirectory + os.path.sep + "Documentation")
print("Changing directory to: " + baseDirectory + os.path.sep +
"Documentation")
print("Calling Doxygen")
subprocessWithImprovedErrors(getCommand("doxygen"), getCommand("doxygen"))
os.chdir(baseDirectory)
print("Changing directory to: " + baseDirectory)
def transformDesignByXslt(designXmlPath, transformPath, outputFile,
additionalParam):
xsltProcPath = os.path.sep.join(['Design', getCommand('saxon')])
additionalParamStringified = [
"{0}={1}".format(key, additionalParam[key])
for key in additionalParam.keys()
]
#pdb.set_trace()
subprocessWithImprovedErrors([
getCommand('java'), '-jar', xsltProcPath, designXmlPath, transformPath,
'-o:' + outputFile
] + additionalParamStringified, getCommand("java"))
def validateDesign(context):
"""Checks design.xml against Design.xsd, and after that performs some additional checks (defined in designValidation.xslt)"""
# 1st line of validation -- does it matches its schema?
# This allows some basic checks
print("1st line of check -- XSD conformance")
print("Validating the file " + designXML + " with the schema " + designXSD)
subprocessWithImprovedErrors([
getCommand("xmllint"), "--noout", "--schema", designPath + designXSD,
designPath + designXML
], getCommand("xmllint"))
# 2nd line of validation -- including XSLT
print("2nd line of check -- more advanced checks using XSLT processor")
transformByKey(TransformKeys.DESIGN_VALIDATION, {'context': context})
def transformDesign(xsltTransformation, outputFile, requiresMerge, astyleRun, additionalParam=None):
"""Generates a file, applying an xslt transform to Design.xml This is done calling saxon9he.jar
Keyword arguments:
xsltTransformation -- xslt file where the transformation is defined
outputFile -- name of the file to be generated
requiresMerge -- if True, will prevent from overwriting output filename, running merge-tool
astyleRun -- if True, will run astyle on generated file
additionalParam -- Optional extra param to be passed e.g. to XSLT transform.
"""
if isinstance(additionalParam, str):
additionalParam = [additionalParam] # compat mode for passing multiple additional params
elif additionalParam == None:
additionalParam = []
# files
XSLT_JAR = '.' + os.path.sep + 'Design' + os.path.sep + getCommand('saxon')
DESIGN_XML = '.' + os.path.sep + 'Design' + os.path.sep + 'Design.xml'
if requiresMerge:
originalOutputFile = outputFile
outputFile = outputFile + '.generated'
try:
# Do transformation
subprocessWithImprovedErrors([getCommand('java'), '-jar', XSLT_JAR, DESIGN_XML, xsltTransformation, '-o:' + outputFile] + additionalParam, getCommand("java"))
if astyleRun:
try:
subprocess.call([getCommand('astyle'), outputFile])
except Exception as e:
try:
subprocess.call([getCommand('indent'), outputFile, '-o', outputFile])
print "We indented your code using 'indent' tool which is a fall-back tool. For best user satisfaction, please install astyle as described in the quasar documentation. "
except Exception as e:
print "We didnt manage to run neither 'astyle' nor 'indent' tool. We're running a fall-back tool now. For best user satisfaction, please install astyle as described in the quasar documentation. "
astyleSubstitute.do_indentation(outputFile)
if requiresMerge:
# If the file existed previously and it is different from the old one we run kdiff3
if (os.path.isfile(originalOutputFile)) and (filecmp.cmp(originalOutputFile, outputFile) == False):
subprocessWithImprovedErrors([getCommand('diff'), "-o", originalOutputFile, originalOutputFile, outputFile], getCommand("diff"), [0, 1]) # 1 is a valid return, since it means that the user quitted without saving the merge, and this is still ok.
else: # If the file didn't exist before, or it is equal to the old one, we rename the generated file to have the proper name
if os.path.isfile(originalOutputFile):
os.remove(originalOutputFile)
os.rename(outputFile, originalOutputFile)
except Exception:
if os.path.isfile(outputFile):
print "Removing partially generated file: {0}".format(outputFile)
os.remove(outputFile) # sometimes the output of XSLT processor is partial...
raise
def generateConfiguration(context):
"""Generates the file Configuration.xsd. This method is called automatically by cmake, it does not need to be called by the user."""
config_xsd_path = os.path.join(context['projectBinaryDir'],'Configuration','Configuration.xsd')
try:
transformByKey( TransformKeys.CONFIGURATION_XSD, {
'context':context,
'metaXsdPath':os.path.join(context['projectSourceDir'],'Meta','config','Meta.xsd').replace('\\','/') } )
except:
if os.path.isfile(config_xsd_path):
os.remove(config_xsd_path)
raise
subprocessWithImprovedErrorsPipeOutputToFile(
[getCommand("xmllint"), "--xinclude", getTransformOutput(TransformKeys.CONFIGURATION_XSD, {'context':context})],
config_xsd_path,
getCommand("xmllint"))
def formatDesign():
"""Formats design.xml. This is done to have always the same indentation format. The formatting is done in a separate file, in case something goes wrong, and then copied over."""
backupName = designXML + ".backup"
tempName = designXML + ".new"
print("Creating a backup of " + designXML + " under the name of " + backupName)
shutil.copyfile(designPath + designXML, designPath + backupName)
print("Formatting the file " + designXML + "using the tool XMLlint. The result will be saved in " + tempName)
try:
if platform.system() == "Windows":
subprocessWithImprovedErrorsPipeOutputToFile([getCommand("xmllint"), designPath + designXML], designPath + tempName, getCommand("xmllint"))
elif platform.system() == "Linux":
subprocessWithImprovedErrorsPipeOutputToFile([getCommand("xmllint"), "--format", designPath + designXML], designPath + tempName, getCommand("xmllint"))
except Exception, e:
raise Exception ("There was a problem formatting the file [" + designXML + "]; Exception: [" + str(e) + "]")
def createDiagram(context, detailLevel=0, mode='dot'):
"""Creates an UML diagram based on the classes of the server.
Keyword arguments:
detailLevel -- Detail level of the diagram. If it is not present, 0 will be assumed
mode -- one of graph layout modes to be passed to graphviz
"""
graphvizArgs = {
'dot': [],
'circo': [
'-Kcirco', '-Gsplines=true', '-Granksep=0.1', '-Gmindist=0',
'-Goverlap=false', '-Gepsilon=0.00001'
],
'fdp': [
'-Kfdp', '-Gsplines=true', '-Goverlap=false', '-Gepsilon=0.00001',
'-GK=0.01', '-Gmaxiter=10000', '-Gstart=random'
]
}
print 'Using {0} as level of detail'.format(str(detailLevel))
print 'Using {0} as layout mode. Hint: from quasar 1.3.5, you can choose among: {1}, run with -h for help.'.format(
mode, ','.join(graphvizArgs.keys()))
transformByKey(TransformKeys.CREATE_DIAGRAM_DOT, {
'context': context,
'detailLevel': detailLevel
})
args = [
"-Tpdf", "-o", designPath + "diagram.pdf",
getTransformOutput(TransformKeys.CREATE_DIAGRAM_DOT,
{'context': context})
] + graphvizArgs[mode]
subprocessWithImprovedErrors([getCommand("graphviz")] + args,
"GraphViz (dot)")
def formatDesign():
"""Formats design.xml. This is done to have always the same indentation format. The formatting is done in a separate file, in case something goes wrong, and then copied over."""
backupName = designXML + ".backup"
tempName = designXML + ".new"
print("Creating a backup of " + designXML + " under the name of " + backupName)
shutil.copyfile(designPath + designXML, designPath + backupName)
print("Formatting the file " + designXML + "using the tool XMLlint. The result will be saved in " + tempName)
try:
if platform.system() == "Windows":
subprocessWithImprovedErrorsPipeOutputToFile([getCommand("xmllint"), designPath + designXML], designPath + tempName, getCommand("xmllint"))
elif platform.system() == "Linux":
subprocessWithImprovedErrorsPipeOutputToFile([getCommand("xmllint"), "--format", designPath + designXML], designPath + tempName, getCommand("xmllint"))
except Exception, e:
raise Exception ("There was a problem formatting the file [" + designXML + "]; Exception: [" + str(e) + "]")
def generateConfiguration():
"""Generates the file Configuration.xsd. This method is called automatically by cmake, it does not need to be called by the user."""
outputFile = os.path.join('Configuration', 'Configuration.xsd')
cleanedOutputFile = os.path.join('Configuration', 'Configuration.xsd.new')
transformationFile = os.path.join(configPath,
"designToConfigurationXSD.xslt")
#output = "Configuration.xsd"
transformDesignVerbose(transformationFile, outputFile, 0, 0)
print("Calling xmllint to modify " + outputFile)
#this call is not using subprocess with improved errors because of the need of the piping.
subprocessWithImprovedErrorsPipeOutputToFile(
[getCommand("xmllint"), "--xinclude", outputFile], cleanedOutputFile,
getCommand("xmllint"))
print("Copying the modified file " + cleanedOutputFile +
" into the name of " + outputFile)
shutil.copyfile(cleanedOutputFile, outputFile)
def checkCompiler():
if platform.system() == "Linux":
checkExecutableExists('make', 'Please, install the package make using the package manager of your distribution.')
return checkExecutableExists('gcc', 'Please, install the package gcc using the package manager of your distribution.', '--help')
#if the system is a windows machine then:
if os.path.isfile(getCommand('vcvarsall')):
printIfVerbose("vcvarsall.bat does exist")
else:
raise Exception("vcvarsall.bat cannot be found in the default path [" + getCommand('vcvarsall') + "]. Maybe Visual Studio 2013 is not installed, or there is a problem with your installation. ")
try:
returnCode = subprocess.call("\"" + getCommand('vcvarsall') + "\"" + ' amd64 && ' + getCommand('msbuild') + ' /h', shell=True, stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
if returnCode == 0:
printIfVerbose("msbuild does exist")
else:
raise Exception("msbuild cannot be properly executed after calling vcvarsall.bat . Maybe Visual Studio 2013 is not installed, or there is a problem with your installation. ")
except:
raise Exception("msbuild cannot be properly executed after calling vcvarsall.bat . Maybe Visual Studio 2013 is not installed, or there is a problem with your installation. ")
def upgradeDesign(additionalParam):
"""Method for adjusting Design.xml for a new Design.xsd when updating to a new version of the Framework"""
print("Formatting your design file ...")
formatDesign()
output = "Design.xml.upgraded"
transformDesignVerbose(designPath + "designToUpgradedDesign.xslt", designPath + output, 0, 0, additionalParam)
print("Formatting the upgraded file ")
formatedOutput = output + ".formatted"
if platform.system() == "Windows":
subprocessWithImprovedErrorsPipeOutputToFile([getCommand("xmllint"), designPath + output], designPath + formatedOutput, getCommand("xmllint"))
elif platform.system() == "Linux":
subprocessWithImprovedErrorsPipeOutputToFile([getCommand("xmllint"), "--format", designPath + output], designPath + formatedOutput, getCommand("xmllint"))
print("Now running merge-tool. Please merge the upgraded changed")
subprocessWithImprovedErrors([getCommand("diff"), "-o", designPath + designXML, designPath + designXML, designPath + formatedOutput], getCommand("diff"))
def upgradeDesign(additionalParam):
"""Method for adjusting Design.xml for a new Design.xsd when updating to a new version of the Framework"""
print("Formatting your design file ...")
formatDesign()
output = "Design.xml.upgraded"
transformDesignVerbose(designPath + "designToUpgradedDesign.xslt", designPath + output, 0, astyleRun=False, additionalParam=additionalParam)
print("Formatting the upgraded file ")
formatedOutput = output + ".formatted"
if platform.system() == "Windows":
subprocessWithImprovedErrorsPipeOutputToFile([getCommand("xmllint"), designPath + output], designPath + formatedOutput, getCommand("xmllint"))
elif platform.system() == "Linux":
subprocessWithImprovedErrorsPipeOutputToFile([getCommand("xmllint"), "--format", designPath + output], designPath + formatedOutput, getCommand("xmllint"))
print("Now running merge-tool. Please merge the upgraded changed")
subprocessWithImprovedErrors([getCommand("diff"), "-o", designPath + designXML, designPath + designXML, designPath + formatedOutput], getCommand("diff"))
def upgradeDesign(additionalParam):
"""Method for adjusting Design.xml for a new Design.xsd when updating to a new version of the Framework"""
print("Formatting your design file ...")
formatDesign()
transformByKey(TransformKeys.UPGRADE_DESIGN, {'whatToDo': additionalParam})
print("Formatting the upgraded file ")
upgradedNonFormatted = getTransformOutput(TransformKeys.UPGRADE_DESIGN)
upgradedFormatted = upgradedNonFormatted + ".formatted"
formatXml(upgradedNonFormatted, upgradedFormatted)
print("Now running merge-tool. Please merge the upgraded changed")
subprocessWithImprovedErrors([
getCommand("diff"), "-o", designPath + designXML,
designPath + designXML, upgradedFormatted
], getCommand("diff"))
def generateCmake(context, buildType="Release"):
"""Generates CMake header lists in various directories, and then calls cmake.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
if not context['projectSourceDir'] == context[
'projectBinaryDir']: # out-of-source build
if os.path.isfile(
os.path.join(context['projectSourceDir'], 'CMakeCache.txt')):
raise Mistake(
'User mistake? CMakeCache.txt exists in source directory; ' +
'that will prevent CMake to make a successful out-of-source build. '
+
'Remove CMakeCache.txt (or attempt "quasar.py clean" and retry'
)
projectSourceDir = context['projectSourceDir']
projectBinaryDir = context['projectBinaryDir']
if not os.path.isdir(projectBinaryDir):
print("PROJECT_BINARY_DIR {0} doesn't exist -- creating it.".format(
projectBinaryDir))
os.mkdir(projectBinaryDir)
os.mkdir(os.path.join(projectBinaryDir, "bin"))
print("Creating symlinks for xml files in the build directory")
symlinkRuntimeDeps(context, '*.xml')
transformByKey(TransformKeys.AS_CMAKE, {'context': context})
transformByKey(TransformKeys.D_CMAKE, {'context': context})
print("Build type [" + buildType + "]")
os.chdir(projectBinaryDir)
print("Calling CMake")
if platform.system() == "Windows":
subprocessWithImprovedErrors([
getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType, "-G",
"Visual Studio 15 2017 Win64", projectSourceDir
], getCommand("cmake"))
elif platform.system() == "Linux":
subprocessWithImprovedErrors([
getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType,
projectSourceDir
], getCommand("cmake"))
def validateDesign():
"""Checks design.xml against Design.xsd, and after that performs some additional checks (defined in designValidation.xslt)"""
# 1st line of validation -- does it matches its schema?
# This allows some basic checks
print("1st line of check -- XSD conformance")
print("Validating the file " + designXML + " with the schema " + designXSD)
try:
subprocessWithImprovedErrors([
getCommand("xmllint"), "--noout", "--schema",
designPath + designXSD, designPath + designXML
], getCommand("xmllint"))
# 2nd line of validation -- including XSLT
print("2nd line of check -- more advanced checks using XSLT processor")
output = "validationOutput.removeme"
transformDesignVerbose(designPath + "designValidation.xslt",
designPath + output, 0, 0)
except Exception, e:
raise Exception("There was a problem validating the file [" +
designXML + "]; Exception: [" + str(e) + "]")
def generateCmake(context, *args):
"""Generates CMake header lists in various directories, and then calls cmake.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
args = list(args) # we do it to be able to remove things via extract_argument, otherwise it is immutable tuple
(args2, builder) = extract_argument(args, "--builder")
if len(args2) > 1:
raise WrongArguments("Max one positional argument for generateCmake")
buildType = "Release" if len(args2) == 0 else args2[0]
if builder is None: builder = BuilderDefault
if not context['projectSourceDir'] == context['projectBinaryDir']: # out-of-source build
if os.path.isfile(os.path.join(context['projectSourceDir'], 'CMakeCache.txt')):
raise Mistake('User mistake? CMakeCache.txt exists in source directory; '+
'that will prevent CMake to make a successful out-of-source build. '+
'Remove CMakeCache.txt and all cmake_install.cmake files, and build/ directory')
projectSourceDir = context['projectSourceDir']
projectBinaryDir = context['projectBinaryDir']
if not os.path.isdir(projectBinaryDir):
print("PROJECT_BINARY_DIR {0} doesn't exist -- creating it.".format(projectBinaryDir))
os.mkdir(projectBinaryDir)
os.mkdir(os.path.join(projectBinaryDir, "bin"))
print("Creating symlinks for xml files in the build directory")
symlinkRuntimeDeps(context, '*.xml')
transformByKey(TransformKeys.AS_CMAKE, {'context':context})
transformByKey(TransformKeys.D_CMAKE, {'context':context})
print("Build type [{0}], Builder [{1}]".format(buildType, builder))
os.chdir(projectBinaryDir)
print("Calling CMake")
if platform.system() == "Windows":
subprocessWithImprovedErrors([getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType,
"-G", "Visual Studio 15 2017 Win64", projectSourceDir],
getCommand("cmake"))
elif platform.system() == "Linux":
builderArgs = [] if builder == BuilderDefault else ["-G", builder]
subprocessWithImprovedErrors([getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType] + builderArgs +
[projectSourceDir],
getCommand("cmake"))
def createDiagram(detailLevel=0):
"""Creates an UML diagram based on the classes of the server.
Keyword arguments:
detailLevel -- Detail level of the diagram. If it is not present, 0 will be assumed
"""
output = "Design.dot"
transformDesignVerbose(designPath + "designToDot.xslt", designPath + output, 0, astyleRun=False, additionalParam="detailLevel=" + str(detailLevel))
print("Generating pdf diagram with dot.")
subprocessWithImprovedErrors([getCommand("graphviz"), "-Tpdf", "-o", designPath + "diagram.pdf", designPath + "Design.dot"], "GraphViz (dot)")
def createDiagram(detailLevel=0):
"""Creates an UML diagram based on the classes of the server.
Keyword arguments:
detailLevel -- Detail level of the diagram. If it is not present, 0 will be assumed
"""
if detailLevel == "":
detailLevel = 0
output = "Design.dot"
transformDesignVerbose(designPath + "designToDot.xslt", designPath + output, 0, 1, "detailLevel=" + str(detailLevel))
print("Generating pdf diagram with dot.")
subprocessWithImprovedErrors([getCommand("graphviz"), "-Tpdf", "-o", designPath + "diagram.pdf", designPath + "Design.dot"], "GraphViz (dot)")
def automatedBuild(buildType="Release",
cmakeToolchainFile="FrameworkInternals" + os.path.sep + "default_configuration.cmake"):
"""Method that generates the cmake headers, and after that calls make/msbuild to compile your server.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
if buildType != "Release" and buildType != "Debug" and cmakeToolchainFile == "FrameworkInternals" + os.path.sep + "default_configuration.cmake":
cmakeToolchainFile = buildType
buildType = "Release"
generateCmake(buildType, cmakeToolchainFile)
print('Calling make/msbuild')
if platform.system() == "Windows":
print('Calling visual studio vcvarsall to set the environment')
print(getCommand("vcvarsall") + ' amd64')
subprocessWithImprovedErrors( "\"" + getCommand("vcvarsall") + '\" amd64', "visual studio vcvarsall.bat")
print('msbuild ALL_BUILD.vcxproj /clp:ErrorsOnly /property:Platform=x64;Configuration=' + buildType)
try:
subprocessWithImprovedErrors( "\"" + getCommand("vcvarsall") + '\" amd64 && ' + getCommand("msbuild") + ' ALL_BUILD.vcxproj /clp:ErrorsOnly /property:Platform=x64;Configuration=' + buildType, "visual studio msbuild")
except Exception, e:
print("Build process error. Exception: [" + str(e) + "]")
def checkKdiff3():
if platform.system() == "Linux":
return checkExecutableExists('diff', 'kdiff3 can be downloaded in http://kdiff3.sourceforge.net/', '--help')
#if the system is a windows machine then:
errorMessageWindows = "kdiff3 cannot be found. Maybe it is not installed, or maybe it is not set in the PATH. \nkdiff3 can be downloaded in http://kdiff3.sourceforge.net/ "
try:
returnCode = subprocess.call(['where', getCommand('diff')], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
if returnCode == 0:
printIfVerbose("kdiff3 does exist")
else:
raise Exception(errorMessageWindows)
except:
raise Exception(errorMessageWindows)
def checkKdiff3():
if platform.system() == "Linux":
return checkExecutableExists('diff', 'kdiff3 can be downloaded in http://kdiff3.sourceforge.net/', '--help')
#if the system is a windows machine then:
errorMessageWindows = "kdiff3 cannot be found. Maybe it is not installed, or maybe it is not set in the PATH. \nkdiff3 can be downloaded in http://kdiff3.sourceforge.net/ "
try:
returnCode = subprocess.call(['where', getCommand('diff')], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
if returnCode == 0:
printIfVerbose("kdiff3 does exist")
else:
raise Exception(errorMessageWindows)
except:
raise Exception(errorMessageWindows)
def generateCmake(projectSourceDir, projectBinaryDir,buildType="Release"):
"""Generates CMake header lists in various directories, and then calls cmake.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
transformDesignVerbose(
os.path.join("AddressSpace", "designToGeneratedCmakeAddressSpace.xslt"),
os.path.join("AddressSpace", "cmake_generated.cmake"),
0,
astyleRun=False,
additionalParam="projectBinaryDir={0}".format(projectBinaryDir ))
transformDesignVerbose(
os.path.join("Device", "designToGeneratedCmakeDevice.xslt"),
os.path.join("Device", "generated", "cmake_header.cmake"),
0,
astyleRun=False,
additionalParam="projectBinaryDir={0}".format(projectBinaryDir ))
print("Build type ["+buildType+"]")
if not os.path.isdir(projectBinaryDir):
print("PROJECT_BINARY_DIR {0} doesn't exist -- creating it.".format(projectBinaryDir))
os.mkdir(projectBinaryDir)
os.chdir(projectBinaryDir)
print("Calling CMake")
if platform.system() == "Windows":
subprocessWithImprovedErrors([getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType,
"-G", "Visual Studio 12 Win64", projectSourceDir],
getCommand("cmake"))
elif platform.system() == "Linux":
subprocessWithImprovedErrors([getCommand("cmake"), "-DCMAKE_BUILD_TYPE=" + buildType,
projectSourceDir],
getCommand("cmake"))
def generateConfiguration(projectBinaryDir):
"""Generates the file Configuration.xsd. This method is called automatically by cmake, it does not need to be called by the user."""
outputFile = os.path.join(projectBinaryDir, 'Configuration',
'Configuration.xsd')
cleanedOutputFile = os.path.join(projectBinaryDir, 'Configuration',
'Configuration.xsd.new')
transformationFile = os.path.join(configPath,
"designToConfigurationXSD.xslt")
transformDesignVerbose(transformationFile,
outputFile,
0,
astyleRun=False,
additionalParam="metaXsdPath={0}".format(
os.path.join(os.getcwd(), "Meta", "config",
"Meta.xsd")))
print("Calling xmllint to modify " + outputFile)
#this call is not using subprocess with improved errors because of the need of the piping.
subprocessWithImprovedErrorsPipeOutputToFile(
[getCommand("xmllint"), "--xinclude", outputFile], cleanedOutputFile,
getCommand("xmllint"))
print("Copying the modified file " + cleanedOutputFile +
" into the name of " + outputFile)
shutil.copyfile(cleanedOutputFile, outputFile)
def validateDesign():
"""Checks design.xml against Design.xsd, and after that performs some additional checks (defined in designValidation.xslt)"""
# 1st line of validation -- does it matches its schema?
# This allows some basic checks
print("1st line of check -- XSD conformance")
print("Validating the file " + designXML + " with the schema " + designXSD)
try:
subprocessWithImprovedErrors([getCommand("xmllint"), "--noout", "--schema", designPath + designXSD, designPath + designXML], getCommand("xmllint"))
# 2nd line of validation -- including XSLT
print("2nd line of check -- more advanced checks using XSLT processor")
output = "validationOutput.removeme"
transformDesignVerbose(designPath + "designValidation.xslt", designPath + output, 0, 0)
except Exception, e:
raise Exception ("There was a problem validating the file [" + designXML + "]; Exception: [" + str(e) + "]")
def checkCompiler():
if platform.system() == "Linux":
checkExecutableExists(
'make',
'Please, install the package make using the package manager of your distribution.'
)
return checkExecutableExists(
'gcc',
'Please, install the package gcc using the package manager of your distribution.',
'--help')
#if the system is a windows machine then:
if os.path.isfile(getCommand('vcvarsall')):
printIfVerbose("vcvarsall.bat does exist")
else:
raise Exception(
"vcvarsall.bat cannot be found in the default path [" +
getCommand('vcvarsall') +
"]. Maybe Visual Studio 2013 is not installed, or there is a problem with your installation. "
)
try:
returnCode = subprocess.call("\"" + getCommand('vcvarsall') + "\"" +
' amd64 && ' + getCommand('msbuild') +
' /h',
shell=True,
stdout=open(os.devnull, 'wb'),
stderr=subprocess.STDOUT)
if returnCode == 0:
printIfVerbose("msbuild does exist")
else:
raise Exception(
"msbuild cannot be properly executed after calling vcvarsall.bat . Maybe Visual Studio 2013 is not installed, or there is a problem with your installation. "
)
except:
raise Exception(
"msbuild cannot be properly executed after calling vcvarsall.bat . Maybe Visual Studio 2013 is not installed, or there is a problem with your installation. "
)
def createDiagram(context, detailLevel=0):
"""Creates an UML diagram based on the classes of the server.
Keyword arguments:
detailLevel -- Detail level of the diagram. If it is not present, 0 will be assumed
"""
if detailLevel == "":
detailLevel = 0
transformByKey(TransformKeys.CREATE_DIAGRAM_DOT, {
'context': context,
'detailLevel': detailLevel
})
print("Generating pdf diagram with dot.")
subprocessWithImprovedErrors([
getCommand("graphviz"), "-Tpdf", "-o", designPath + "diagram.pdf",
getTransformOutput(TransformKeys.CREATE_DIAGRAM_DOT,
{'context': context})
], "GraphViz (dot)")
def distClean(*args, **kwargs):
if len(args) > 0:
param = args[0]
"""
Cleaning method. It first calls msbuild/make clean and after that it will delete the leftover generated files in the Server.
If parameter --orig is specified, the generated files ending with the extension .orig will also be cleared.
"""
if platform.system() == "Windows":
print('Calling visual studio vcvarsall to set the environment')
print(getCommand("vcvarsall") + ' amd64')
subprocessWithImprovedErrors(
"\"" + getCommand("vcvarsall") + '\" amd64',
"visual studio vcvarsall.bat")
print('Calling msbuild clean')
print('msbuild ALL_BUILD.vcxproj /t:Clean')
subprocessWithImprovedErrors(
"\"" + getCommand("vcvarsall") + '\" amd64 && ' +
getCommand("msbuild") + ' ALL_BUILD.vcxproj /t:Clean',
"visual studio msbuild", [0, 1])
elif platform.system() == "Linux":
subprocessWithImprovedErrors([getCommand('make'), 'clean'],
getCommand("make"))
print('Deleting generated files and directories')
deleteFolderRecursively('.', 'CMakeFiles')
deleteFolderRecursively('.', 'Win32')
deleteFolderRecursively('.', 'Debug')
deleteFolderRecursively('.', 'Release')
deleteFolderRecursively('.', 'x64')
deleteFolderRecursively('.', '*.dir')
deleteExtensionRecursively('.', 'vcxproj')
deleteExtensionRecursively('.', 'sln')
deleteExtensionRecursively('.', 'vcxproj.filters')
deleteExtensionRecursively('.', 'vcxproj.user')
deleteExtensionRecursively('.', 'sdf')
deleteExtensionRecursively('.', 'opensdf')
deleteFileRecursively('.', 'CMakeCache.txt')
deleteFileRecursively('.', 'Makefile')
deleteFileRecursively('.', 'cmake_install.cmake')
deleteFileRecursively('.', 'cmake_generated.cmake')
deleteFileRecursively('.', 'generated_files_list.cmake')
deleteExtensionRecursively('.', 'pyc')
if param == '--orig':
print('Removing .orig files')
deleteExtensionRecursively('.', 'orig')
def distClean(param=None):
"""
Cleaning method. It first calls msbuild/make clean and after that it will delete the leftover generated files in the Server.
If parameter --orig is specified, the generated files ending with the extension .orig will also be cleared.
"""
if platform.system() == "Windows":
print('Calling visual studio vcvarsall to set the environment')
print(getCommand("vcvarsall") + ' amd64')
subprocessWithImprovedErrors("\"" + getCommand("vcvarsall") + '\" amd64', "visual studio vcvarsall.bat")
print('Calling msbuild clean')
print('msbuild ALL_BUILD.vcxproj /t:Clean')
subprocessWithImprovedErrors("\"" + getCommand("vcvarsall") + '\" amd64 && ' + getCommand("msbuild") + ' ALL_BUILD.vcxproj /t:Clean', "visual studio msbuild", [0, 1])
elif platform.system() == "Linux":
subprocessWithImprovedErrors([getCommand('make'), 'clean'], getCommand("make"))
print('Deleting generated files and directories')
deleteFolderRecursively('.', 'CMakeFiles')
deleteFolderRecursively('.', 'Win32')
deleteFolderRecursively('.', 'Debug')
deleteFolderRecursively('.', 'Release')
deleteFolderRecursively('.', 'x64')
deleteFolderRecursively('.', '*.dir')
deleteExtensionRecursively('.', 'vcxproj')
deleteExtensionRecursively('.', 'sln')
deleteExtensionRecursively('.', 'vcxproj.filters')
deleteExtensionRecursively('.', 'vcxproj.user')
deleteExtensionRecursively('.', 'sdf')
deleteExtensionRecursively('.', 'opensdf')
deleteFileRecursively('.', 'CMakeCache.txt')
deleteFileRecursively('.', 'Makefile')
deleteFileRecursively('.', 'cmake_install.cmake')
deleteFileRecursively('.', 'cmake_generated.cmake')
deleteFileRecursively('.', 'generated_files_list.cmake')
deleteExtensionRecursively('.', 'pyc')
if param == '--orig':
print('Removing .orig files')
deleteExtensionRecursively('.', 'orig')
def transformDesign(xsltTransformation, outputFile, overwriteProtection, astyleRun, additionalParam=None):
"""Generates a file, applying an xslt transform to Design.xml This is done calling saxon9he.jar
Keyword arguments:
xsltTransformation -- xslt file where the transformation is defined
outputFile -- name of the file to be generated
overwriteProtection -- if 1, will prevent from overwriting output filename, running merge-tool
astyleRun -- if 1, will run astyle on generated file
additionalParam -- Optional extra param. If specified, it will be given directly to saxon9he.jar
"""
#files
XSLT_JAR = '.' + os.path.sep + 'Design' + os.path.sep + getCommand('saxon')
DESIGN_XML = '.' + os.path.sep + 'Design' + os.path.sep + 'Design.xml'
if overwriteProtection == 1:
originalOutputFile = outputFile
outputFile = outputFile + '.generated'
try:
#Do transformation
if additionalParam == None:
subprocessWithImprovedErrors([getCommand('java'), '-jar', XSLT_JAR, DESIGN_XML, xsltTransformation, '-o:' + outputFile], getCommand("java"))
else:
subprocessWithImprovedErrors([getCommand('java'), '-jar', XSLT_JAR, DESIGN_XML, xsltTransformation, '-o:' + outputFile, additionalParam], getCommand("java"))
if astyleRun == 1:
subprocessWithImprovedErrors([getCommand('astyle'), outputFile], getCommand("astyle"))
if overwriteProtection == 1:
#If the file existed previously and it is different from the old one we run kdiff3
if (os.path.isfile(originalOutputFile)) and (filecmp.cmp(originalOutputFile, outputFile) == False):
subprocessWithImprovedErrors([getCommand('diff'), "-o", originalOutputFile, originalOutputFile, outputFile], getCommand("diff"), [0, 1])#1 is a valid return, since it means that the user quitted without saving the merge, and this is still ok.
else:#If the file didn't exist before, or it is equal to the old one, we rename the generated file to have the proper name
if os.path.isfile(originalOutputFile):
os.remove(originalOutputFile)
os.rename(outputFile, originalOutputFile)
except Exception, e:
raise Exception ("There was a problem generating file [" + outputFile + "]; Exception: [" + str(e) + "]")
def transformDesign(xsltTransformation,
outputFile,
requiresMerge,
astyleRun,
additionalParam=None):
"""Generates a file, applying a transform (XSLT or Jinja2) to Design.xml
Keyword arguments:
transformPath -- transform file where the transformation is defined, either XSLT or Jinja2
outputFile -- name of the file to be generated
requiresMerge -- if True, will prevent from overwriting output filename, running merge-tool
astyleRun -- if True, will run astyle on generated file
additionalParam -- Optional extra param to be passed e.g. to XSLT transform.
"""
transformPath = xsltTransformation
newAdditionalParam = {}
if isinstance(additionalParam,
list): # this is the legacy mode for certain quasar modules
processedAdditionalParam = {}
for chunk in additionalParam:
[key, value] = chunk.split('=')
processedAdditionalParam[key] = value
elif additionalParam == None:
processedAdditionalParam = {}
else:
processedAdditionalParam = additionalParam
# files
designXmlPath = '.' + os.path.sep + 'Design' + os.path.sep + 'Design.xml' # TODO os.path.join
if requiresMerge:
originalOutputFile = outputFile
outputFile = outputFile + '.generated'
try:
if transformPath.endswith('.jinja'):
transformDesignByJinja(designXmlPath, transformPath, outputFile,
processedAdditionalParam)
elif transformPath.endswith('.xslt'):
transformDesignByXslt(designXmlPath, transformPath, outputFile,
processedAdditionalParam)
else:
raise Exception("Couldnt determine transformation type")
if astyleRun:
try:
subprocess.call([getCommand('astyle'), outputFile])
except Exception as e:
try:
subprocess.call(
[getCommand('indent'), outputFile, '-o', outputFile])
print(
"We indented your code using 'indent' tool which is a fall-back tool. For best user satisfaction, please install astyle as described in the quasar documentation. "
)
except Exception as e:
print(
"We didnt manage to run neither 'astyle' nor 'indent' tool. We're running a fall-back tool now. For best user satisfaction, please install astyle as described in the quasar documentation. "
)
astyleSubstitute.do_indentation(outputFile)
if requiresMerge:
# If the file existed previously and it is different from the old one we run kdiff3
if (os.path.isfile(originalOutputFile)) and (filecmp.cmp(
originalOutputFile, outputFile) == False):
subprocessWithImprovedErrors(
[
getCommand('diff'), "-o", originalOutputFile,
originalOutputFile, outputFile
], getCommand("diff"), [0, 1]
) # 1 is a valid return, since it means that the user quitted without saving the merge, and this is still ok.
else: # If the file didn't exist before, or it is equal to the old one, we rename the generated file to have the proper name
if os.path.isfile(originalOutputFile):
os.remove(originalOutputFile)
os.rename(outputFile, originalOutputFile)
except Exception:
if os.path.isfile(outputFile):
print("Removing partially generated file: {0}".format(outputFile))
os.remove(outputFile
) # sometimes the output of XSLT processor is partial...
raise
)
generateCmake(projectSourceDir, projectBinaryDir, buildType)
print('Calling make/msbuild')
if platform.system() == "Windows":
print('Calling visual studio vcvarsall to set the environment')
print(getCommand("vcvarsall") + ' amd64')
subprocessWithImprovedErrors(
"\"" + getCommand("vcvarsall") + '\" amd64',
"visual studio vcvarsall.bat")
print(
'msbuild ALL_BUILD.vcxproj /clp:ErrorsOnly /property:Platform=x64;Configuration='
+ buildType)
try:
subprocessWithImprovedErrors(
"\"" + getCommand("vcvarsall") + '\" amd64 && ' +
getCommand("msbuild") +
' ALL_BUILD.vcxproj /clp:ErrorsOnly /property:Platform=x64;Configuration='
+ buildType, "visual studio msbuild")
except Exception, e:
print("Build process error. Exception: [" + str(e) + "]")
elif platform.system() == "Linux":
print('make -j$(nproc)')
#we call process nproc and store its output
process = subprocess.Popen(["nproc"], stdout=subprocess.PIPE)
out, err = process.communicate()
#this output is used for calling make
subprocessWithImprovedErrors(
[getCommand("make"), "-j" + str(int(out))], getCommand("make")
) #the conversion from string to int and back to string is to remove all whitespaces and ensure that we have an integer
def automatedBuild(buildType="Release",
cmakeToolchainFile="FrameworkInternals" + os.path.sep + "default_configuration.cmake"):
"""Method that generates the cmake headers, and after that calls make/msbuild to compile your server.
Keyword arguments:
buildType -- Optional parameter to specify Debug or Release build. If it is not specified it will default to Release.
"""
if buildType != "Release" and buildType != "Debug" and cmakeToolchainFile == "FrameworkInternals" + os.path.sep + "default_configuration.cmake":
cmakeToolchainFile = buildType
buildType = "Release"
generateCmake(buildType, cmakeToolchainFile)
print('Calling make/msbuild')
if platform.system() == "Windows":
print('Calling visual studio vcvarsall to set the environment')
print(getCommand("vcvarsall") + ' amd64')
subprocessWithImprovedErrors( "\"" + getCommand("vcvarsall") + '\" amd64', "visual studio vcvarsall.bat")
print('msbuild ALL_BUILD.vcxproj /clp:ErrorsOnly /property:Platform=x64;Configuration=' + buildType)
try:
subprocessWithImprovedErrors( "\"" + getCommand("vcvarsall") + '\" amd64 && ' + getCommand("msbuild") + ' ALL_BUILD.vcxproj /clp:ErrorsOnly /property:Platform=x64;Configuration=' + buildType, "visual studio msbuild")
except Exception, e:
print("Build process error. Exception: [" + str(e) + "]")
elif platform.system() == "Linux":
print('make -j$(nproc)')
#we call process nproc and store its output
process = subprocess.Popen(["nproc"], stdout=subprocess.PIPE)
out, err = process.communicate()
#this output is used for calling make
subprocessWithImprovedErrors([getCommand("make"), "-j" + str(int(out))], getCommand("make"))#the conversion from string to int and back to string is to remove all whitespaces and ensure that we have an integer
