def copyContentFromPath(sourcePath, contentPath, folderName):
# Check folderName
if not folderName or not cc.legalName(folderName):
raise cc.RTAMError(cc.invalidFolderNameMsg, cc.invalidFolderNameCode)
# Copy the source tree
try:
sourcePath = os.path.abspath(sourcePath)
shutil.copytree(sourcePath, contentPath + os.path.sep + folderName)
except:
raise cc.RTAMError(cc.errorCopyingContentMsg,
cc.errorCopyingContentCode)
return
def checkWorkingDirectory(context):
# Check whether the tool has a working directory
workingDirectory = context.get("workingDirectory", None)
if not workingDirectory or not os.path.isdir(workingDirectory):
raise cc.RTAMError(cc.noDirectoryMsg, cc.noDirectoryCode)
return os.path.abspath(workingDirectory)
def getPortRange(project):
# Check project.
if not project or not isinstance(project, dict):
raise cc.RTAMError(invalidProjectMsg, invalidProjectCode)
# Get the port range.
portRange = project.get("portRange", [5001, 5500])
return portRange
def test_RTAMError(self):
message = None
errorCode = None
try:
raise cnc.RTAMError(cnc.noDirectorySpecMsg,
cnc.noDirectorySpecCode)
except cnc.RTAMError as err:
message = err.message
errorCode = err.errorCode
self.assertTrue(message == cnc.noDirectorySpecMsg
and errorCode == cnc.noDirectorySpecCode)
def initBuild(project, context):
# Check project & context.
if not project or not isinstance(project, dict):
raise cc.RTAMError(cc.invalidProjectMsg, cc.invalidProjectCode)
if not context or not isinstance(context, dict):
raise cc.RTAMError(cc.invalidContextMsg, cc.invalidContextCode)
# Check whether the base image has been specified.
baseImage = project.get("baseImage", context.get("baseImage", None))
if not baseImage:
raise cc.RTAMError(cc.noBaseImageMsg, cc.noBaseImageCode)
# Get the project path and delete it if it exists
workingPath = cc.getWorkingPath(project, context)
if os.path.isdir(workingPath):
shutil.rmtree(workingPath)
os.makedirs(workingPath)
return workingPath
def copySourceFromPath(sourcePath, foldersPath, folderName):
# Copy the source tree
try:
sourcePath = os.path.abspath(sourcePath)
# TODO: SIMPLIFY THIS!
os.mkdir(foldersPath + os.path.sep + folderName)
folderPath = getFolderPath(foldersPath, folderName)
#baseName = os.path.basename(sourcePath)
#shutil.copytree(sourcePath, folderPath + os.path.sep + baseName)
shutil.copytree(sourcePath, folderPath + os.path.sep + folderName)
except:
raise cc.RTAMError(cc.errorCopyingSourceMsg, cc.errorCopyingSourceCode)
return
def buildProject(project, basePath, context):
# TODO: CONFIGURE TLS FOR NGINX BASED ON THE serviceTLS PARAMETER
# Get the project name
projectName = project.get('projectName', None)
if not projectName:
raise cc.RTAMError(noProjectNameMsg, noProjectNameCode)
# Initialize the build
workingPath = initBuild(project, context)
foldersPath = workingPath + os.path.sep + 'folders'
os.makedirs(foldersPath)
logPath = workingPath + os.path.sep + 'logs'
os.makedirs(logPath)
os.makedirs(logPath + os.path.sep + 'smp')
os.makedirs(logPath + os.path.sep + 'smi')
cannrHome = '/usr/local/cannr'
# String for Dockerfile
dockerText = getDockerfile()
# Main script for static startup
mainText = buildCodeLine(0, ['# Startup script', '\n'])
mainText += buildCodeLine(0, ['# Start workers'])
# Create container
# Import base runtime image
baseImage = project.get("baseImage", context.get("baseImage", None))
dockerText = dockerText.replace('<base image>', baseImage)
# Label with maintainer
maintainerEmail = context.get("maintainerEmail", None)
if maintainerEmail:
dockerText = dockerText.replace(
'#<maintainer>', 'LABEL maintainer="' + maintainerEmail + '"')
else:
dockerText = dockerText.replace('#<maintainer>',
'# No maintainer information')
# R and Python packages to import, respectively.
rPackageNames = []
pPackageNames = []
pPackageMap = {} # Map of package names to package name/version stringss
# Get the port range and first port.
portRange = getPortRange(project)
port = portRange[0]
pWorkers = project.get('workers', 2)
# Create the event calendar for the SMP and add events to start the SMI and NGINX.
smiPath = project.get('smiPath', '/web/smi.py')
nginxPath = project.get('nginxPath', '/etc/nginx')
smiPort = project.get("smiPort", 8080)
nginxPort = project.get("nginxPort", 80)
#eventCalendar = EventCalendar()
#eventCalendar.addEntry(None, "startSMI", {"path": smiPath, "port": smiPort}, 3)
#eventCalendar.addEntry(None, "startNGINX", {"path": nginxPath, "port": nginxPort}, 1)
# NGINX config.
ngnixHttpBlock = 'http {\n'
ngnixHttpBlock += '\t' + 'include /etc/nginx/mime.types;\n'
# include /etc/nginx/mime.types;
# NGINX server block
nginxServerBlock = '\t' + 'server {\n'
# Add limit on body size, if applicable
maxBodySize = project.get('maxBodySize', None)
if maxBodySize:
nginxServerBlock += 2 * '\t' + 'client_max_body_size\t' + maxBodySize + ';\n'
# Whether running locally or in a container.
local = context.get('local', False)
# Loop through folders in the project, add to project. Main loop!
#folderNames = cc.getFolderNames(project)
folderNames = cc.getCodeFolderNames(project)
for folderName in folderNames:
# Get the folder and copy the source files to the new folder
folder = cc.getFolder(folderName, project)
# Check for source path
# TODO: CHANGE TO USE sourcePath IF local ELSE /external/projects/ ONLY IF projectsPath UNDEFINED IN context.
#sourcePath = folder.get("sourcePath", None) if local else '/external/projects/' + projectName + '/' + folderName
sourcePath = folder.get("sourcePath", None) if local else os.path.join(
'/projects', projectName, folderName)
#projectsPath = context.get('projectsPath', None)
#projectsPath = projectsPath if projectsPath else '/external/projects'
#sourcePath = folder.get("sourcePath", None) if local else os.path.join(projectsPath, projectName, folderName)
if not sourcePath:
raise cc.RTAMError(cc.noSourcePathMsg, cc.noSourcePathCode)
# Adjust if not absolute path
sp = Path(sourcePath)
if not sp.is_absolute():
# str(Path(basePath).resolve())
sp = Path(str(Path(basePath).resolve()) + os.path.sep + sourcePath)
sourcePath = str(sp.resolve())
# Copy the source files
copySourceFromPath(sourcePath, foldersPath, folderName)
# Create the log directory
folderLogPath = logPath + os.path.sep + 'workers' + os.path.sep + folderName
# Get the number of workers for the folder
workers = folder.get('workers', pWorkers)
# If R
if folder.get("language", "Python") == "R":
# Loop through modules in the folder
moduleNames = cc.getModuleNames(folder)
for moduleName in moduleNames:
module = cc.getModule(moduleName, folder)
# Generate runtime file for the module and copy to service home
sourceFileName = module.get("sourceFile", None)
if not sourceFileName:
raise cc.RTAMError(missingSourceFileNameMsg,
missingSourceFileNameCode)
# Change to the working directory in the script.
sourceText = '# Change to the source directory\n'
folderPath = '/folders/' + folderName
#sourceText += buildCodeLine(0, ['setwd("', cc.getHome(folderName, folder), '")\n'])
sourceText += buildCodeLine(
0, ['setwd("', folderPath + '/' + folderName, '")\n'])
# Read the source file and append it.
with open(sourcePath + os.path.sep + sourceFileName,
"r") as sourceFile:
sourceText += sourceFile.read()
# Add in the Plumber wrappers.
moduleText = sourceText + 2 * '\n' + buildRModuleEpilogue(
folderName, moduleName, project)
folderPath = getFolderPath(foldersPath, folderName)
modulePath = folderPath + os.path.sep + moduleName + ".R"
# Write out the module script.
with open(modulePath, "w") as moduleFile:
moduleFile.write(moduleText)
# Add to NGINX config file and startup event calendar of SMP
upstreamName = folderName + '_' + moduleName
ngnixHttpBlock += '\t' + 'upstream ' + upstreamName + ' {\n'
# Add workers
workerID = 1
path = '/folders/' + folderName + '/' + moduleName + '.R'
for worker in range(workers):
ngnixHttpBlock += 2 * '\t' + 'server localhost:' + str(
port) + ' max_conns=1;\n'
#eventCalendar.addEntry(None, "startPlumber", {"folder": folderName, "module": moduleName, "path": modulePath, "port": port}, 1)
# TODO: THROW EXCEPTION IF PORT OUT OF RANGE
mainText += buildCodeLine(0, [
'Rscript', ' --vanilla ', cannrHome + '/runApp.R', ' ',
path, ' ',
str(port), ' ',
str(workerID), ' &'
])
port += 1
workerID += 1
ngnixHttpBlock += '\t' + '}\n'
# Add location to NGINX server block.
nginxServerBlock += 2 * '\t' + 'location /services/' + folderName + '/' + moduleName + ' {\n'
nginxServerBlock += 3 * '\t' + 'proxy_pass http://' + upstreamName + ';\n'
nginxServerBlock += 2 * '\t' + '}\n'
# Add packages to list of R packages
packages = module.get("packages", None)
if packages:
rPackageNames.extend(packages)
os.makedirs(folderLogPath + os.path.sep + moduleName)
# TODO:
# Add help for module
# Else if Python
else:
# Generate runtime file for the folder and copy to service home
moduleText = buildPyFolder(folderName, project)
folderPath = getFolderPath(foldersPath, folderName)
modulePath = folderPath + os.path.sep + folderName + ".py"
with open(modulePath, "w") as moduleFile:
moduleFile.write(moduleText)
# Add to NGINX config file and startup event calendar of SMP
ngnixHttpBlock += '\t' + 'upstream ' + folderName + ' {\n'
# Add workers
path = '/folders/' + folderName + '/' + folderName + '.py'
for worker in range(workers):
ngnixHttpBlock += 2 * '\t' + 'server localhost:' + str(
port) + ' max_conns=1;\n'
#eventCalendar.addEntry(None, "startFlask", {"folder": folderName, "path": modulePath, "port": port}, 1)
# TODO: THROW EXCEPTION IF PORT OUT OF RANGE
mainText += buildCodeLine(
0, ['python ', path, ' ',
str(port), ' &'])
port += 1
ngnixHttpBlock += '\t' + '}\n'
# Add location to NGINX server block.
nginxServerBlock += 2 * '\t' + 'location /services/' + folderName + ' {\n'
nginxServerBlock += 3 * '\t' + 'proxy_pass http://' + folderName + ';\n'
nginxServerBlock += 2 * '\t' + '}\n'
# Loop through modules in the folder
moduleNames = cc.getModuleNames(folder)
for moduleName in moduleNames:
module = cc.getModule(moduleName, folder)
# Add packages to list of Python packages
packages = module.get("packages", None)
if packages:
pPackageNames.extend(packages)
# Create the log directory
os.makedirs(folderLogPath)
# TODO:
# Add help for module
# Handle static content
# Create the directory for static content
contentPath = workingPath + os.path.sep + 'content/web'
#try:
# os.makedirs(contentPath)
#except:
# pass
#content = project.get('content', {})
contentFolderNames = cc.getContentFolderNames(project)
for folderName in contentFolderNames:
# Add location for the content to the NGINX server block.
nginxServerBlock += 2 * '\t' + 'location /web/' + folderName + ' {\n'
nginxServerBlock += 3 * '\t' + 'root /content;\n'
nginxServerBlock += 2 * '\t' + '}\n'
# Copy the content into the project
#folder = content.get(folderName)
# Get the folder and copy the source files to the new folder
folder = cc.getFolder(folderName, project)
#sourcePath = folder.get('sourcePath', None)
sourcePath = folder.get("sourcePath", None) if local else os.path.join(
'/projects', projectName, folderName)
if sourcePath:
sp = Path(sourcePath)
if not sp.is_absolute():
sp = Path(
str(Path(basePath).resolve()) + os.path.sep + str(sp))
sourcePath = str(sp.resolve())
copyContentFromPath(sourcePath, contentPath, folderName)
# Close NGINX http server block and add it to the http block.
nginxServerBlock += '\t' + '}\n'
ngnixHttpBlock += nginxServerBlock
ngnixHttpBlock += '}'
# Save NGINX http block to conf.d/http
nginxConfPath = nginxPath + os.path.sep + 'conf.d'
nginxProjectConfPath = workingPath + os.path.sep + 'conf.d'
try:
os.makedirs(nginxProjectConfPath)
except:
pass
nginxProjectHttpPath = nginxProjectConfPath + os.path.sep + 'http'
with open(nginxProjectHttpPath, "w") as httpFile:
httpFile.write(ngnixHttpBlock)
# Add imports of R packages to container and Dockerfile
installText = ''
rPackageSet = set(rPackageNames)
if len(rPackageSet):
for pkg in rPackageSet:
if not cc.isRInstPkg(pkg):
installText += 'RUN install2.r ' + pkg + '\n'
dockerText = dockerText.replace('#<R Packages>', installText)
else:
dockerText = dockerText.replace('#<R Packages>',
'# No R packages to install')
# Build list of imports of Python packages
requirementsText = ''
pPackageSet = set(pPackageNames)
pPackageList = []
for pkg in pPackageSet:
if not cc.isStdPkg(pkg) and not cc.isInstPkg(pkg):
pPackageList.append(pkg)
pPackageMap = cc.buildPPackMap(pPackageList)
for packageName in pPackageMap:
requirementsText += pPackageMap[packageName] + '\n'
# Copy project file to project directory
with open(os.path.join(workingPath, 'requirements.txt'),
"w") as requirementsFile:
requirementsFile.write(requirementsText)
# Copy static content into container
dockerContentText = ''
for folderName in contentFolderNames:
dockerContentText += 'COPY ./content/web/' + folderName + ' /content\n'
if dockerContentText:
dockerText = dockerText.replace('#<Static Content>', dockerContentText)
else:
dockerText = dockerText.replace('#<Static Content>',
'# No static content')
# Number the nodes in the project
project = walkNumber(project)
# Copy project file to project directory
with open(workingPath + os.path.sep + 'project.json', "w") as projectFile:
projectFile.write(json.dumps(project, indent=2))
# Add command to start NGINX
mainText += buildCodeLine(0, [])
mainText += buildCodeLine(0, ['# Start NGINX'])
mainText += buildCodeLine(0, ["nginx -g 'daemon off;'"])
# Copy startup script to project directory
with open(workingPath + os.path.sep + 'main.sh', "w") as mainFile:
mainFile.write(mainText)
# Write initial event calendar to file.
#os.chdir(workingPath)
#os.mkdir('eventCalendar')
#eventCalendar.write(workingPath + os.path.sep + 'eventCalendar')
# Write out Dockerfile to the project directory
with open(workingPath + os.path.sep + 'Dockerfile', "w") as dockerFile:
dockerFile.write(dockerText)
return
def getFolderPath(foldersPath, folderName):
try:
return foldersPath.replace('/', os.path.sep) + os.path.sep + folderName
except:
raise cc.RTAMError(cc.folderPathErrorMsg, cc.folderPathErrorCode)
