Exemplo n.º 1
0
def copyContentFromPath(sourcePath, contentPath, folderName):

    # Check folderName
    if not folderName or not cc.legalName(folderName):
        raise cc.RTAMError(cc.invalidFolderNameMsg, cc.invalidFolderNameCode)

    # Copy the source tree
    try:
        sourcePath = os.path.abspath(sourcePath)
        shutil.copytree(sourcePath, contentPath + os.path.sep + folderName)
    except:
        raise cc.RTAMError(cc.errorCopyingContentMsg,
                           cc.errorCopyingContentCode)

    return
Exemplo n.º 2
0
def checkWorkingDirectory(context):
    # Check whether the tool has a working directory
    workingDirectory = context.get("workingDirectory", None)
    if not workingDirectory or not os.path.isdir(workingDirectory):
        raise cc.RTAMError(cc.noDirectoryMsg, cc.noDirectoryCode)

    return os.path.abspath(workingDirectory)
Exemplo n.º 3
0
def getPortRange(project):

    # Check project.
    if not project or not isinstance(project, dict):
        raise cc.RTAMError(invalidProjectMsg, invalidProjectCode)

    # Get the port range.
    portRange = project.get("portRange", [5001, 5500])
    return portRange
Exemplo n.º 4
0
    def test_RTAMError(self):
        message = None
        errorCode = None
        try:
            raise cnc.RTAMError(cnc.noDirectorySpecMsg,
                                cnc.noDirectorySpecCode)
        except cnc.RTAMError as err:
            message = err.message
            errorCode = err.errorCode

        self.assertTrue(message == cnc.noDirectorySpecMsg
                        and errorCode == cnc.noDirectorySpecCode)
Exemplo n.º 5
0
def initBuild(project, context):

    # Check project & context.
    if not project or not isinstance(project, dict):
        raise cc.RTAMError(cc.invalidProjectMsg, cc.invalidProjectCode)

    if not context or not isinstance(context, dict):
        raise cc.RTAMError(cc.invalidContextMsg, cc.invalidContextCode)

    # Check whether the base image has been specified.
    baseImage = project.get("baseImage", context.get("baseImage", None))
    if not baseImage:
        raise cc.RTAMError(cc.noBaseImageMsg, cc.noBaseImageCode)

    # Get the project path and delete it if it exists
    workingPath = cc.getWorkingPath(project, context)
    if os.path.isdir(workingPath):
        shutil.rmtree(workingPath)

    os.makedirs(workingPath)

    return workingPath
Exemplo n.º 6
0
def copySourceFromPath(sourcePath, foldersPath, folderName):

    # Copy the source tree
    try:
        sourcePath = os.path.abspath(sourcePath)
        # TODO:  SIMPLIFY THIS!
        os.mkdir(foldersPath + os.path.sep + folderName)
        folderPath = getFolderPath(foldersPath, folderName)
        #baseName = os.path.basename(sourcePath)
        #shutil.copytree(sourcePath, folderPath + os.path.sep + baseName)
        shutil.copytree(sourcePath, folderPath + os.path.sep + folderName)
    except:
        raise cc.RTAMError(cc.errorCopyingSourceMsg, cc.errorCopyingSourceCode)

    return
Exemplo n.º 7
0
def buildProject(project, basePath, context):

    # TODO: CONFIGURE TLS FOR NGINX BASED ON THE serviceTLS PARAMETER

    # Get the project name
    projectName = project.get('projectName', None)
    if not projectName:
        raise cc.RTAMError(noProjectNameMsg, noProjectNameCode)

    # Initialize the build
    workingPath = initBuild(project, context)
    foldersPath = workingPath + os.path.sep + 'folders'
    os.makedirs(foldersPath)
    logPath = workingPath + os.path.sep + 'logs'
    os.makedirs(logPath)
    os.makedirs(logPath + os.path.sep + 'smp')
    os.makedirs(logPath + os.path.sep + 'smi')

    cannrHome = '/usr/local/cannr'

    # String for Dockerfile
    dockerText = getDockerfile()

    # Main script for static startup
    mainText = buildCodeLine(0, ['# Startup script', '\n'])
    mainText += buildCodeLine(0, ['# Start workers'])

    # Create container

    # Import base runtime image
    baseImage = project.get("baseImage", context.get("baseImage", None))
    dockerText = dockerText.replace('<base image>', baseImage)

    # Label with maintainer
    maintainerEmail = context.get("maintainerEmail", None)
    if maintainerEmail:
        dockerText = dockerText.replace(
            '#<maintainer>', 'LABEL maintainer="' + maintainerEmail + '"')
    else:
        dockerText = dockerText.replace('#<maintainer>',
                                        '# No maintainer information')

    # R and Python packages to import, respectively.
    rPackageNames = []
    pPackageNames = []
    pPackageMap = {}  # Map of package names to package name/version stringss

    # Get the port range and first port.
    portRange = getPortRange(project)
    port = portRange[0]
    pWorkers = project.get('workers', 2)

    # Create the event calendar for the SMP and add events to start the SMI and NGINX.
    smiPath = project.get('smiPath', '/web/smi.py')
    nginxPath = project.get('nginxPath', '/etc/nginx')
    smiPort = project.get("smiPort", 8080)
    nginxPort = project.get("nginxPort", 80)
    #eventCalendar = EventCalendar()
    #eventCalendar.addEntry(None, "startSMI", {"path": smiPath, "port": smiPort}, 3)
    #eventCalendar.addEntry(None, "startNGINX", {"path": nginxPath, "port": nginxPort}, 1)

    # NGINX config.
    ngnixHttpBlock = 'http {\n'
    ngnixHttpBlock += '\t' + 'include /etc/nginx/mime.types;\n'

    #    include /etc/nginx/mime.types;

    # NGINX server block
    nginxServerBlock = '\t' + 'server {\n'

    # Add limit on body size, if applicable
    maxBodySize = project.get('maxBodySize', None)
    if maxBodySize:
        nginxServerBlock += 2 * '\t' + 'client_max_body_size\t' + maxBodySize + ';\n'

    # Whether running locally or in a container.
    local = context.get('local', False)

    # Loop through folders in the project, add to project.  Main loop!
    #folderNames = cc.getFolderNames(project)
    folderNames = cc.getCodeFolderNames(project)
    for folderName in folderNames:

        # Get the folder and copy the source files to the new folder
        folder = cc.getFolder(folderName, project)

        # Check for source path
        # TODO:  CHANGE TO USE sourcePath IF local ELSE /external/projects/ ONLY IF projectsPath UNDEFINED IN context.
        #sourcePath = folder.get("sourcePath", None) if local else '/external/projects/' + projectName + '/' + folderName
        sourcePath = folder.get("sourcePath", None) if local else os.path.join(
            '/projects', projectName, folderName)
        #projectsPath = context.get('projectsPath', None)
        #projectsPath = projectsPath if projectsPath else '/external/projects'
        #sourcePath = folder.get("sourcePath", None) if local else os.path.join(projectsPath, projectName, folderName)
        if not sourcePath:
            raise cc.RTAMError(cc.noSourcePathMsg, cc.noSourcePathCode)

        # Adjust if not absolute path
        sp = Path(sourcePath)
        if not sp.is_absolute():
            # str(Path(basePath).resolve())
            sp = Path(str(Path(basePath).resolve()) + os.path.sep + sourcePath)
            sourcePath = str(sp.resolve())

        # Copy the source files
        copySourceFromPath(sourcePath, foldersPath, folderName)

        # Create the log directory
        folderLogPath = logPath + os.path.sep + 'workers' + os.path.sep + folderName

        # Get the number of workers for the folder
        workers = folder.get('workers', pWorkers)

        # If R
        if folder.get("language", "Python") == "R":

            # Loop through modules in the folder
            moduleNames = cc.getModuleNames(folder)
            for moduleName in moduleNames:
                module = cc.getModule(moduleName, folder)

                # Generate runtime file for the module and copy to service home
                sourceFileName = module.get("sourceFile", None)
                if not sourceFileName:
                    raise cc.RTAMError(missingSourceFileNameMsg,
                                       missingSourceFileNameCode)

                # Change to the working directory in the script.
                sourceText = '# Change to the source directory\n'
                folderPath = '/folders/' + folderName
                #sourceText += buildCodeLine(0, ['setwd("', cc.getHome(folderName, folder), '")\n'])
                sourceText += buildCodeLine(
                    0, ['setwd("', folderPath + '/' + folderName, '")\n'])

                # Read the source file and append it.
                with open(sourcePath + os.path.sep + sourceFileName,
                          "r") as sourceFile:
                    sourceText += sourceFile.read()

                # Add in the Plumber wrappers.
                moduleText = sourceText + 2 * '\n' + buildRModuleEpilogue(
                    folderName, moduleName, project)
                folderPath = getFolderPath(foldersPath, folderName)
                modulePath = folderPath + os.path.sep + moduleName + ".R"

                # Write out the module script.
                with open(modulePath, "w") as moduleFile:
                    moduleFile.write(moduleText)

                # Add to NGINX config file and startup event calendar of SMP
                upstreamName = folderName + '_' + moduleName
                ngnixHttpBlock += '\t' + 'upstream ' + upstreamName + ' {\n'

                # Add workers
                workerID = 1
                path = '/folders/' + folderName + '/' + moduleName + '.R'
                for worker in range(workers):
                    ngnixHttpBlock += 2 * '\t' + 'server localhost:' + str(
                        port) + ' max_conns=1;\n'
                    #eventCalendar.addEntry(None, "startPlumber", {"folder": folderName, "module": moduleName, "path": modulePath, "port": port}, 1)
                    # TODO: THROW EXCEPTION IF PORT OUT OF RANGE
                    mainText += buildCodeLine(0, [
                        'Rscript', ' --vanilla ', cannrHome + '/runApp.R', ' ',
                        path, ' ',
                        str(port), ' ',
                        str(workerID), ' &'
                    ])
                    port += 1
                    workerID += 1

                ngnixHttpBlock += '\t' + '}\n'

                # Add location to NGINX server block.
                nginxServerBlock += 2 * '\t' + 'location /services/' + folderName + '/' + moduleName + ' {\n'
                nginxServerBlock += 3 * '\t' + 'proxy_pass http://' + upstreamName + ';\n'
                nginxServerBlock += 2 * '\t' + '}\n'

                # Add packages to list of R packages
                packages = module.get("packages", None)
                if packages:
                    rPackageNames.extend(packages)

                os.makedirs(folderLogPath + os.path.sep + moduleName)

                # TODO:
                # Add help for module

        # Else if Python
        else:

            # Generate runtime file for the folder and copy to service home
            moduleText = buildPyFolder(folderName, project)
            folderPath = getFolderPath(foldersPath, folderName)
            modulePath = folderPath + os.path.sep + folderName + ".py"

            with open(modulePath, "w") as moduleFile:
                moduleFile.write(moduleText)

            # Add to NGINX config file and startup event calendar of SMP
            ngnixHttpBlock += '\t' + 'upstream ' + folderName + ' {\n'

            # Add workers
            path = '/folders/' + folderName + '/' + folderName + '.py'
            for worker in range(workers):
                ngnixHttpBlock += 2 * '\t' + 'server localhost:' + str(
                    port) + ' max_conns=1;\n'
                #eventCalendar.addEntry(None, "startFlask", {"folder": folderName, "path": modulePath, "port": port}, 1)
                # TODO: THROW EXCEPTION IF PORT OUT OF RANGE
                mainText += buildCodeLine(
                    0, ['python ', path, ' ',
                        str(port), ' &'])
                port += 1

            ngnixHttpBlock += '\t' + '}\n'

            # Add location to NGINX server block.
            nginxServerBlock += 2 * '\t' + 'location /services/' + folderName + ' {\n'
            nginxServerBlock += 3 * '\t' + 'proxy_pass http://' + folderName + ';\n'
            nginxServerBlock += 2 * '\t' + '}\n'

            # Loop through modules in the folder
            moduleNames = cc.getModuleNames(folder)
            for moduleName in moduleNames:
                module = cc.getModule(moduleName, folder)

                # Add packages to list of Python packages
                packages = module.get("packages", None)
                if packages:
                    pPackageNames.extend(packages)

            # Create the log directory
            os.makedirs(folderLogPath)

            # TODO:
            # Add help for module

    # Handle static content
    # Create the directory for static content
    contentPath = workingPath + os.path.sep + 'content/web'
    #try:
    #    os.makedirs(contentPath)
    #except:
    #    pass
    #content = project.get('content', {})
    contentFolderNames = cc.getContentFolderNames(project)
    for folderName in contentFolderNames:
        # Add location for the content to the NGINX server block.
        nginxServerBlock += 2 * '\t' + 'location /web/' + folderName + ' {\n'
        nginxServerBlock += 3 * '\t' + 'root /content;\n'
        nginxServerBlock += 2 * '\t' + '}\n'
        # Copy the content into the project
        #folder = content.get(folderName)

        # Get the folder and copy the source files to the new folder
        folder = cc.getFolder(folderName, project)

        #sourcePath = folder.get('sourcePath', None)
        sourcePath = folder.get("sourcePath", None) if local else os.path.join(
            '/projects', projectName, folderName)
        if sourcePath:
            sp = Path(sourcePath)
            if not sp.is_absolute():
                sp = Path(
                    str(Path(basePath).resolve()) + os.path.sep + str(sp))
                sourcePath = str(sp.resolve())
            copyContentFromPath(sourcePath, contentPath, folderName)

    # Close NGINX http server block and add it to the http block.
    nginxServerBlock += '\t' + '}\n'
    ngnixHttpBlock += nginxServerBlock
    ngnixHttpBlock += '}'

    # Save NGINX http block to conf.d/http
    nginxConfPath = nginxPath + os.path.sep + 'conf.d'
    nginxProjectConfPath = workingPath + os.path.sep + 'conf.d'
    try:
        os.makedirs(nginxProjectConfPath)
    except:
        pass
    nginxProjectHttpPath = nginxProjectConfPath + os.path.sep + 'http'

    with open(nginxProjectHttpPath, "w") as httpFile:
        httpFile.write(ngnixHttpBlock)

    # Add imports of R packages to container and Dockerfile
    installText = ''
    rPackageSet = set(rPackageNames)
    if len(rPackageSet):
        for pkg in rPackageSet:
            if not cc.isRInstPkg(pkg):
                installText += 'RUN install2.r ' + pkg + '\n'
        dockerText = dockerText.replace('#<R Packages>', installText)
    else:
        dockerText = dockerText.replace('#<R Packages>',
                                        '# No R packages to install')

    # Build list of imports of Python packages
    requirementsText = ''
    pPackageSet = set(pPackageNames)
    pPackageList = []
    for pkg in pPackageSet:
        if not cc.isStdPkg(pkg) and not cc.isInstPkg(pkg):
            pPackageList.append(pkg)
    pPackageMap = cc.buildPPackMap(pPackageList)
    for packageName in pPackageMap:
        requirementsText += pPackageMap[packageName] + '\n'

    # Copy project file to project directory
    with open(os.path.join(workingPath, 'requirements.txt'),
              "w") as requirementsFile:
        requirementsFile.write(requirementsText)

    # Copy static content into container
    dockerContentText = ''
    for folderName in contentFolderNames:
        dockerContentText += 'COPY ./content/web/' + folderName + ' /content\n'
    if dockerContentText:
        dockerText = dockerText.replace('#<Static Content>', dockerContentText)
    else:
        dockerText = dockerText.replace('#<Static Content>',
                                        '# No static content')

    # Number the nodes in the project
    project = walkNumber(project)

    # Copy project file to project directory
    with open(workingPath + os.path.sep + 'project.json', "w") as projectFile:
        projectFile.write(json.dumps(project, indent=2))

    # Add command to start NGINX
    mainText += buildCodeLine(0, [])
    mainText += buildCodeLine(0, ['# Start NGINX'])
    mainText += buildCodeLine(0, ["nginx -g 'daemon off;'"])

    # Copy startup script to project directory
    with open(workingPath + os.path.sep + 'main.sh', "w") as mainFile:
        mainFile.write(mainText)

    # Write initial event calendar to file.
    #os.chdir(workingPath)
    #os.mkdir('eventCalendar')
    #eventCalendar.write(workingPath + os.path.sep + 'eventCalendar')

    # Write out Dockerfile to the project directory
    with open(workingPath + os.path.sep + 'Dockerfile', "w") as dockerFile:
        dockerFile.write(dockerText)

    return
Exemplo n.º 8
0
def getFolderPath(foldersPath, folderName):
    try:
        return foldersPath.replace('/', os.path.sep) + os.path.sep + folderName
    except:
        raise cc.RTAMError(cc.folderPathErrorMsg, cc.folderPathErrorCode)