def setUp(self): self.context = cnc.readJSONFile('../source/runtime/context.json') if platform.system() == 'Windows': self.project = cnc.readJSONFile( '../examples/project1/winproject.json') else: self.project = cnc.readJSONFile( '../examples/project1/project.json') self.webProject = cnc.readJSONFile('../source/webtool/project.json') with open('testfiles/upload.dat', 'rb') as uploadFile: self.uploadData = uploadFile.read() self.jsonData = ' { "data" :\n[ [1, 2, 3],\n[4, 5, 6]\n]\n}' self.dfJSONData = ' { "data" :\n[ {"a": 1, "b": 2, "c": 3},\n{"a": 4, "b": 5, "c": 6}\n]\n} \n' self.df = pandas.read_json( '[{"a": 1, "b": 2, "c": 3},{"a": 4, "b": 5, "c": 6}]') self.dfJSON = '[{"a":1,"b":2,"c":3},{"a":4,"b":5,"c":6}]' self.dictJSONData = ' { "data" :\n { "stuff1": [ {"a": 1, "b": 2, "c": 3},\n{"a": 4, "b": 5, "c": 6}\n], "stuff2": 8 }\n} \n' self.dictJSON = json.dumps( json.loads(self.dictJSONData).get('data', None))
def setUp(self): self.context = cnc.readJSONFile('context.json') if platform.system() == 'Windows': self.project = cnc.readJSONFile('../examples/project1/winproject.json') else: self.project = cnc.readJSONFile('../examples/project1/project.json') pass
def buildFromFile(path, context): project = cc.readJSONFile(path) basePath = os.path.dirname(path) buildProject(project, basePath, context) return
def test_initBuild(self): project = cnc.readJSONFile('../examples/project1/project.json') context = cnc.readJSONFile('context.json') with self.assertRaises(cnc.RTAMError) as result: cnb.initBuild(None, context) with self.assertRaises(cnc.RTAMError) as result: cnb.initBuild(project, None) with self.assertRaises(cnc.RTAMError) as result: cnb.initBuild([], context) with self.assertRaises(cnc.RTAMError) as result: cnb.initBuild(project, []) project['projectName'] = None with self.assertRaises(cnc.RTAMError) as result: cnb.initBuild(project, context) project = cnc.readJSONFile('../examples/project1/project.json') project['baseImage'] = None context['baseImage'] = None with self.assertRaises(cnc.RTAMError) as result: cnb.initBuild(project, context) project = cnc.readJSONFile('../examples/project1/project.json') context = cnc.readJSONFile('context.json') context['workingDirectory'] = '../doesnotexist' with self.assertRaises(cnc.RTAMError) as result: cnb.initBuild(project, context) context = cnc.readJSONFile('context.json') workingDir = cnb.initBuild(project, context) checkPath = os.path.abspath('working/project1') self.assertEqual(workingDir, checkPath)
def deleteFolder_(input): try: # Get the project from the input, return error if no project projectName = input.get('projectName', None) folderName = input.get('folderName', None) if not projectName or not folderName: return { 'succeeded': False, 'error': 'noProjectName', 'errorMsg': 'Project or folder not specified' } ''' # Check to make sure /projects exists if not projectsPath: return {'succeeded': False, 'error': 'noProjectsDir', 'errorMsg': 'No projects directory'} ''' # Project directory and project file path projectPath = projectsPath + '/' + projectName filePath = projectPath + '/project.json' # Retrieve the project project = cc.readJSONFile(filePath) # Check to see if the folder exists folderPath = projectPath + '/' + folderName if os.path.isdir(folderPath): # Remove the folder and its subfolders from the file system shutil.rmtree(folderPath) # Remove the folder from the project folders = project.get('folders', None) if not folders: project.put('folders', {}) else: folders.pop(folderName, None) # Write out project file with open(filePath, 'w') as projectFile: projectFile.write(json.dumps(project, indent=2)) # Return the updated project return {'succeeded': True, 'project': project} except Exception as err: return { 'succeeded': False, 'error': 'errorDeletingFolder', 'errorMsg': 'Error deleting folder', 'detail': str(err) }
def getProjectsDict(): # Get the collection of projects projects = {} # List the contents of the projects directory for projectName in os.listdir(projectsPath): # Check that the item is a directory projectDir = os.path.join(projectsPath, projectName) if os.path.isdir(projectDir): # Get the project file path and check that the file exists filePath = os.path.join(projectDir, 'project.json') if os.path.isfile(filePath): # Read the file and add it to the collection project = cc.readJSONFile(filePath) projects[projectName] = project return projects
def test_saveProject(self): project = cnc.readJSONFile('../examples/project1/project.json') authPolicy = self.context.get('authPolicy') authPolicyLen = authPolicy.get('authPolicyLen') authPolicyChars = authPolicy.get('authPolicyChars') result = cnc.saveProject(self.project, 'project1.json', authPolicyLen, authPolicyChars) self.assertEqual(result, 1) authentication = {'password': '******', 'adminID': 'IRule'} project['authentication'] = authentication result = cnc.saveProject(project, 'project1.json', authPolicyLen, authPolicyChars) self.assertEqual(result, 3) authentication = {'password': '******', 'adminID': 'IRule'} project['authentication'] = authentication result = cnc.saveProject(project, 'project1.json', authPolicyLen, authPolicyChars) self.assertEqual(result, 0)
""" CANNR TM analytics container building tool delete/overwrite confirm. Returns directory that could be deleted or overwritten by the build process. Copyright 2020 Pat Tendick [email protected] All rights reserved Maintainer Pat Tendick [email protected] """ import cannrcore as cc import sys project = cc.readJSONFile(sys.argv[1]) context = cc.readJSONFile(sys.argv[2]) #print('The directory') print(cc.getProjectPath(project, context)) #print('and its descendants will be deleted or overwritten.')
def startWorkers(context): projectFilePath = context.get('projectFilePath', '/project.json') cannrHome = context.get('cannrHome', '/cannr') foldersPath = '/' try: project = cnr.readJSONFile(projectFilePath) if project: # Get port info portRange = project.get('portRange', [5000, 10000]) port = portRange[0] maxPort = portRange[1] numPorts = maxPort - port # Check if there are enough ports if countPorts(project) > numPorts: logging.error('Not enough ports:', numPorts) return False # Go through the folders and start worker processes rProcesses = context.get('rProcesses') pProcesses = context.get('pProcesses') folders = project.get('folders', None) for folderName in folders: folder = folders.get('folderName', {}) folderPath = cnr.getFolderPath(foldersPath, folderName) if folder.get('language', 'Python') == 'R' and folder.get( 'modules', None): modules = folder.get('modules', None) for moduleName in modules: try: module = modules.get(moduleName) path = folderPath + os.path.sep + moduleName + '.R' # Add workers workers = folder.get('workers', 1) for workerNum in range(workers): processInfo = { 'path': path, 'port': port, 'folder': folderName, 'module': moduleName, 'workerNum': workerNum } # Record the information about the process and save it in the context rProcess = openProcess(processInfo, [ 'Rscript', '--vanilla', cannrHome + os.path.sep + 'runApp.R', path, str(port), str(workerNum) ]) rProcesses.append(rProcess) port += 1 # Log that the process was started logging.info('R worker for module ' + moduleName + ' in folder ' + folderName + ' started on port ' + str(port)) except Error as err: # Log error logging.exception( 'Error starting R worker for module ' + moduleName + ' in folder ' + folderName + ' on port ' + str(port) + ':', err) else: try: path = folderPath + os.path.sep + folderName + ".py" # Add workers workers = folder.get('workers', 1) for workerNum in range(workers): processInfo = { 'path': path, 'port': port, 'folder': folderName, 'workerNum': workerNum } # Record the information about the process and save it in the context pProcess = openProcess( processInfo, ['python', path, str(port), str(workerNum)]) context.get('pProcesses').append(pProcess) port += 1 # Log that the process was started logging.info('Python worker for folder ' + folderName + ' started on port ' + str(port)) except Error as err: # Log error logging.exception( 'Error starting Python worker for folder ' + folderName + ' on port ' + str(port) + ':', err) return True else: # Log error logging.error('Unable to start workers: Missing configuration') except Error as err: # Log error logging.exception('Error starting workers:', err) return False
def setUp(self): shutil.rmtree('../projects') os.mkdir('../projects') self.context = cnc.readJSONFile('context.json') svs.setContext(self.context) self.project1 = { 'projectTitle': 'Basic test project', 'projectName': 'project1', 'projectDescription': 'project1 description', 'baseImage': 'cannr-base', 'maintainerName': 'Pat Tendick', 'maintainerEmail': '*****@*****.**', 'dependencyNotice': 'Copyright/license notice for dependencies', 'portRange': [5001, 5500], 'nginxPort': 80, 'smiPort': 8080, 'initRequired': True, 'local': True } self.project2 = { 'projectTitle': 'Basic test project', 'projectName': 'project2', 'projectDescription': 'project2 description', 'baseImage': 'cannr-base', 'maintainerName': 'Pat Tendick', 'maintainerEmail': '*****@*****.**', 'dependencyNotice': 'Copyright/license notice for dependencies', 'portRange': [5001, 5500], 'nginxPort': 80, 'smiPort': 8080, 'initRequired': True, 'local': True, 'folders': { 'folder1': { 'folderTitle': 'folder1' }, 'folder2': { 'folderTitle': 'folder2' } } } self.project3 = { 'projectTitle': 'Modified test project', 'projectName': 'project1', 'projectDescription': 'project1 description', 'baseImage': 'cannr-base', 'maintainerName': 'Pat Tendick', 'maintainerEmail': '*****@*****.**', 'dependencyNotice': 'Copyright/license notice for dependencies', 'portRange': [5001, 5500], 'nginxPort': 80, 'smiPort': 8080, 'initRequired': True, 'local': True } # Read a Python source file with open('../source/base_image/cannr/lib/cannrcore.py', "r") as sourceFile: self.pSource = sourceFile.read() # Read an R source file with open('../examples/project1/folder2/iris.R', "r") as sourceFile: self.rSource = sourceFile.read() # Read the source file with open('testfiles/upload.dat', "rb") as sourceFile: self.uploadData = sourceFile.read() # Read the source file with open('testfiles/folder123.zip', "rb") as sourceFile: self.uploadZip = sourceFile.read()
def renameProject(input): try: # Convert input to dictionary #inputObject = json.loads(input) # Get the project from the input, return error if no project oldProjectName = input.get('oldProjectName', None) newProjectName = input.get('newProjectName', None) if not oldProjectName or not newProjectName: return { 'succeeded': False, 'error': 'noProjectName', 'errorMsg': 'No project names given' } # TODO: CHECK THAT newProjectName IS LEGAL ''' # Check to make sure /projects exists if not projectsPath: return {'succeeded': False, 'error': 'noProjectsDir', 'errorMsg': 'No projects directory'} ''' # Check to see if the project exists already oldProjectPath = projectsPath + '/' + oldProjectName if not os.path.isdir(oldProjectPath): # If not, create it return { 'succeeded': False, 'error': 'noProject', 'errorMsg': 'Project does not exist' } # Check to make sure a project with the new name doesn't exist newProjectPath = projectsPath + '/' + newProjectName if os.path.isdir(newProjectPath): # If not, create it return { 'succeeded': False, 'error': 'projectExists', 'errorMsg': 'A project with the new name exists' } # Retrieve the project document filePath = oldProjectPath + '/project.json' project = cc.readJSONFile(filePath) project['projectName'] = newProjectName # Rename the project folder os.rename(oldProjectPath, newProjectPath) # Write the project file to the project directory with open(newProjectPath + '/project.json', 'w') as projectFile: projectFile.write(json.dumps(project, indent=2)) # Return the return {'succeeded': True, 'project': project} except Exception as err: return { 'succeeded': False, 'error': 'errorUpdatingProject', 'errorMsg': 'Error updating project', 'detail': str(err) }
def upload(resourceNames, data, uploadType='folder'): # Get project name and folder name from resourceNames projectName = resourceNames.get('projectname', None) folderName = resourceNames.get('foldername', None) fileName = resourceNames.get('filename', None) # Check that project name and folder name are OK if not (projectName and cc.legalName(projectName) and folderName and cc.legalName(folderName)): return { 'succeeded': False, 'error': 'badProjectFolderName', 'errorMsg': 'Missing or invalid project or folder name', 'detail': 'Project name: ' + (projectName if projectName else '') + '\n' + 'Folder name: ' + (folderName if folderName else '') } # Check that project directory and project exist projectPath = projectsPath + '/' + projectName filePath = projectPath + '/project.json' if not (os.path.isdir(projectPath) and os.path.isfile(filePath)): return { 'succeeded': False, 'error': 'projectNotExist', 'errorMsg': 'Project does not exist' } # Get the project object from the project directory project = cc.readJSONFile(filePath) if not project: return { 'succeeded': False, 'error': 'projectNotExist', 'errorMsg': 'Project does not exist' } # Get the data from the request #data = request.get_data() # Write out the files newFileNames = [] fullFolderName = projectPath + '/' + folderName + '/' uploadType = uploadType.lower() try: if uploadType == 'file': newFileName = ci.writeFile(data, fullFolderName, fileName) if newFileName: newFileNames.append(newFileName) elif uploadType == 'zipfile': newFileNames = ci.writeZipFiles(data, fullFolderName) else: newFileNames = ci.writeFiles(data, fullFolderName) except Exception as err: return { 'succeeded': False, 'error': 'errorWritingFiles', 'errorMsg': 'Error writing files', 'detail': str(err) } # Get existing folder, if any, add if none folders = project.get('folders', None) if not folders: folders = {} project['folders'] = folders folder = folders.get(folderName, None) if not folder: folder = {} folders[folderName] = folder # Get the list of new file names in the top level directory of the folder newFileNames = ci.regexFilter( [s[len(fullFolderName):] for s in newFileNames], '^[^/]*$') # Add to the existing list if appropriate fileNames = folder.get('fileNames', []) if uploadType == 'file': fileNames = fileNames.extend(newFileNames) else: fileNames = newFileNames # Remove duplicates and update in the project fileNames = list(set(fileNames)) folder['fileNames'] = fileNames # Write the updated project to the project folder and return the project. return {'succeeded': True, 'project': project}
import traceback import re import shutil import docker from flask import request from werkzeug.utils import secure_filename # Try to read the context context = None #dockerURL = None #if os.path.exists('context.json'): # context = cc.readJSONFile('context.json') #if os.path.exists('/config/context.json'): # context = cc.readJSONFile('/config/context.json') if os.path.exists('/config/context.json'): context = cc.readJSONFile('/config/context.json') else: # TODO: THIS IS AN ERROR! pass # Try to define the projects folder and working directory, create if necessary if context: projectsPath = '/projects' workingDirectory = '/working' #dockerURL = context.get('dockerURL', 'unix://var/run/docker.sock') osPlatform = context.get('osPlatform', 'Darwin') baseImage = context.get('baseImage', None) if not os.path.isdir(projectsPath): os.mkdir(projectsPath)
def test_walkNumber(self): projectFilePath = os.path.abspath('../examples/project1/project.json') project = cnc.readJSONFile(projectFilePath) cnb.walkNumber(project) self.assertTrue(project['folders']['pyfolder']['modules']['sum']['nodeNumber'])