def setUp(self):
self.context = cnc.readJSONFile('../source/runtime/context.json')
if platform.system() == 'Windows':
self.project = cnc.readJSONFile(
'../examples/project1/winproject.json')
else:
self.project = cnc.readJSONFile(
'../examples/project1/project.json')
self.webProject = cnc.readJSONFile('../source/webtool/project.json')
with open('testfiles/upload.dat', 'rb') as uploadFile:
self.uploadData = uploadFile.read()
self.jsonData = ' { "data" :\n[ [1, 2, 3],\n[4, 5, 6]\n]\n}'
self.dfJSONData = ' { "data" :\n[ {"a": 1, "b": 2, "c": 3},\n{"a": 4, "b": 5, "c": 6}\n]\n} \n'
self.df = pandas.read_json(
'[{"a": 1, "b": 2, "c": 3},{"a": 4, "b": 5, "c": 6}]')
self.dfJSON = '[{"a":1,"b":2,"c":3},{"a":4,"b":5,"c":6}]'
self.dictJSONData = ' { "data" :\n { "stuff1": [ {"a": 1, "b": 2, "c": 3},\n{"a": 4, "b": 5, "c": 6}\n], "stuff2": 8 }\n} \n'
self.dictJSON = json.dumps(
json.loads(self.dictJSONData).get('data', None))
def setUp(self):
self.context = cnc.readJSONFile('context.json')
if platform.system() == 'Windows':
self.project = cnc.readJSONFile('../examples/project1/winproject.json')
else:
self.project = cnc.readJSONFile('../examples/project1/project.json')
pass
def buildFromFile(path, context):
project = cc.readJSONFile(path)
basePath = os.path.dirname(path)
buildProject(project, basePath, context)
return
def test_initBuild(self):
project = cnc.readJSONFile('../examples/project1/project.json')
context = cnc.readJSONFile('context.json')
with self.assertRaises(cnc.RTAMError) as result:
cnb.initBuild(None, context)
with self.assertRaises(cnc.RTAMError) as result:
cnb.initBuild(project, None)
with self.assertRaises(cnc.RTAMError) as result:
cnb.initBuild([], context)
with self.assertRaises(cnc.RTAMError) as result:
cnb.initBuild(project, [])
project['projectName'] = None
with self.assertRaises(cnc.RTAMError) as result:
cnb.initBuild(project, context)
project = cnc.readJSONFile('../examples/project1/project.json')
project['baseImage'] = None
context['baseImage'] = None
with self.assertRaises(cnc.RTAMError) as result:
cnb.initBuild(project, context)
project = cnc.readJSONFile('../examples/project1/project.json')
context = cnc.readJSONFile('context.json')
context['workingDirectory'] = '../doesnotexist'
with self.assertRaises(cnc.RTAMError) as result:
cnb.initBuild(project, context)
context = cnc.readJSONFile('context.json')
workingDir = cnb.initBuild(project, context)
checkPath = os.path.abspath('working/project1')
self.assertEqual(workingDir, checkPath)
def deleteFolder_(input):
try:
# Get the project from the input, return error if no project
projectName = input.get('projectName', None)
folderName = input.get('folderName', None)
if not projectName or not folderName:
return {
'succeeded': False,
'error': 'noProjectName',
'errorMsg': 'Project or folder not specified'
}
'''
# Check to make sure /projects exists
if not projectsPath:
return {'succeeded': False, 'error': 'noProjectsDir', 'errorMsg': 'No projects directory'}
'''
# Project directory and project file path
projectPath = projectsPath + '/' + projectName
filePath = projectPath + '/project.json'
# Retrieve the project
project = cc.readJSONFile(filePath)
# Check to see if the folder exists
folderPath = projectPath + '/' + folderName
if os.path.isdir(folderPath):
# Remove the folder and its subfolders from the file system
shutil.rmtree(folderPath)
# Remove the folder from the project
folders = project.get('folders', None)
if not folders:
project.put('folders', {})
else:
folders.pop(folderName, None)
# Write out project file
with open(filePath, 'w') as projectFile:
projectFile.write(json.dumps(project, indent=2))
# Return the updated project
return {'succeeded': True, 'project': project}
except Exception as err:
return {
'succeeded': False,
'error': 'errorDeletingFolder',
'errorMsg': 'Error deleting folder',
'detail': str(err)
}
def getProjectsDict():
# Get the collection of projects
projects = {}
# List the contents of the projects directory
for projectName in os.listdir(projectsPath):
# Check that the item is a directory
projectDir = os.path.join(projectsPath, projectName)
if os.path.isdir(projectDir):
# Get the project file path and check that the file exists
filePath = os.path.join(projectDir, 'project.json')
if os.path.isfile(filePath):
# Read the file and add it to the collection
project = cc.readJSONFile(filePath)
projects[projectName] = project
return projects
def test_saveProject(self):
project = cnc.readJSONFile('../examples/project1/project.json')
authPolicy = self.context.get('authPolicy')
authPolicyLen = authPolicy.get('authPolicyLen')
authPolicyChars = authPolicy.get('authPolicyChars')
result = cnc.saveProject(self.project, 'project1.json', authPolicyLen,
authPolicyChars)
self.assertEqual(result, 1)
authentication = {'password': '******', 'adminID': 'IRule'}
project['authentication'] = authentication
result = cnc.saveProject(project, 'project1.json', authPolicyLen,
authPolicyChars)
self.assertEqual(result, 3)
authentication = {'password': '******', 'adminID': 'IRule'}
project['authentication'] = authentication
result = cnc.saveProject(project, 'project1.json', authPolicyLen,
authPolicyChars)
self.assertEqual(result, 0)
""" CANNR TM analytics container building tool delete/overwrite confirm. Returns directory that could be deleted or overwritten by the build process. Copyright 2020 Pat Tendick [email protected] All rights reserved Maintainer Pat Tendick [email protected] """ import cannrcore as cc import sys project = cc.readJSONFile(sys.argv[1]) context = cc.readJSONFile(sys.argv[2]) #print('The directory') print(cc.getProjectPath(project, context)) #print('and its descendants will be deleted or overwritten.')
def startWorkers(context):
projectFilePath = context.get('projectFilePath', '/project.json')
cannrHome = context.get('cannrHome', '/cannr')
foldersPath = '/'
try:
project = cnr.readJSONFile(projectFilePath)
if project:
# Get port info
portRange = project.get('portRange', [5000, 10000])
port = portRange[0]
maxPort = portRange[1]
numPorts = maxPort - port
# Check if there are enough ports
if countPorts(project) > numPorts:
logging.error('Not enough ports:', numPorts)
return False
# Go through the folders and start worker processes
rProcesses = context.get('rProcesses')
pProcesses = context.get('pProcesses')
folders = project.get('folders', None)
for folderName in folders:
folder = folders.get('folderName', {})
folderPath = cnr.getFolderPath(foldersPath, folderName)
if folder.get('language', 'Python') == 'R' and folder.get(
'modules', None):
modules = folder.get('modules', None)
for moduleName in modules:
try:
module = modules.get(moduleName)
path = folderPath + os.path.sep + moduleName + '.R'
# Add workers
workers = folder.get('workers', 1)
for workerNum in range(workers):
processInfo = {
'path': path,
'port': port,
'folder': folderName,
'module': moduleName,
'workerNum': workerNum
}
# Record the information about the process and save it in the context
rProcess = openProcess(processInfo, [
'Rscript', '--vanilla',
cannrHome + os.path.sep + 'runApp.R', path,
str(port),
str(workerNum)
])
rProcesses.append(rProcess)
port += 1
# Log that the process was started
logging.info('R worker for module ' +
moduleName + ' in folder ' +
folderName + ' started on port ' +
str(port))
except Error as err:
# Log error
logging.exception(
'Error starting R worker for module ' +
moduleName + ' in folder ' + folderName +
' on port ' + str(port) + ':', err)
else:
try:
path = folderPath + os.path.sep + folderName + ".py"
# Add workers
workers = folder.get('workers', 1)
for workerNum in range(workers):
processInfo = {
'path': path,
'port': port,
'folder': folderName,
'workerNum': workerNum
}
# Record the information about the process and save it in the context
pProcess = openProcess(
processInfo,
['python', path,
str(port),
str(workerNum)])
context.get('pProcesses').append(pProcess)
port += 1
# Log that the process was started
logging.info('Python worker for folder ' +
folderName + ' started on port ' +
str(port))
except Error as err:
# Log error
logging.exception(
'Error starting Python worker for folder ' +
folderName + ' on port ' + str(port) + ':', err)
return True
else:
# Log error
logging.error('Unable to start workers: Missing configuration')
except Error as err:
# Log error
logging.exception('Error starting workers:', err)
return False
def setUp(self):
shutil.rmtree('../projects')
os.mkdir('../projects')
self.context = cnc.readJSONFile('context.json')
svs.setContext(self.context)
self.project1 = {
'projectTitle': 'Basic test project',
'projectName': 'project1',
'projectDescription': 'project1 description',
'baseImage': 'cannr-base',
'maintainerName': 'Pat Tendick',
'maintainerEmail': '*****@*****.**',
'dependencyNotice': 'Copyright/license notice for dependencies',
'portRange': [5001, 5500],
'nginxPort': 80,
'smiPort': 8080,
'initRequired': True,
'local': True
}
self.project2 = {
'projectTitle': 'Basic test project',
'projectName': 'project2',
'projectDescription': 'project2 description',
'baseImage': 'cannr-base',
'maintainerName': 'Pat Tendick',
'maintainerEmail': '*****@*****.**',
'dependencyNotice': 'Copyright/license notice for dependencies',
'portRange': [5001, 5500],
'nginxPort': 80,
'smiPort': 8080,
'initRequired': True,
'local': True,
'folders': {
'folder1': {
'folderTitle': 'folder1'
},
'folder2': {
'folderTitle': 'folder2'
}
}
}
self.project3 = {
'projectTitle': 'Modified test project',
'projectName': 'project1',
'projectDescription': 'project1 description',
'baseImage': 'cannr-base',
'maintainerName': 'Pat Tendick',
'maintainerEmail': '*****@*****.**',
'dependencyNotice': 'Copyright/license notice for dependencies',
'portRange': [5001, 5500],
'nginxPort': 80,
'smiPort': 8080,
'initRequired': True,
'local': True
}
# Read a Python source file
with open('../source/base_image/cannr/lib/cannrcore.py',
"r") as sourceFile:
self.pSource = sourceFile.read()
# Read an R source file
with open('../examples/project1/folder2/iris.R', "r") as sourceFile:
self.rSource = sourceFile.read()
# Read the source file
with open('testfiles/upload.dat', "rb") as sourceFile:
self.uploadData = sourceFile.read()
# Read the source file
with open('testfiles/folder123.zip', "rb") as sourceFile:
self.uploadZip = sourceFile.read()
def renameProject(input):
try:
# Convert input to dictionary
#inputObject = json.loads(input)
# Get the project from the input, return error if no project
oldProjectName = input.get('oldProjectName', None)
newProjectName = input.get('newProjectName', None)
if not oldProjectName or not newProjectName:
return {
'succeeded': False,
'error': 'noProjectName',
'errorMsg': 'No project names given'
}
# TODO: CHECK THAT newProjectName IS LEGAL
'''
# Check to make sure /projects exists
if not projectsPath:
return {'succeeded': False, 'error': 'noProjectsDir', 'errorMsg': 'No projects directory'}
'''
# Check to see if the project exists already
oldProjectPath = projectsPath + '/' + oldProjectName
if not os.path.isdir(oldProjectPath):
# If not, create it
return {
'succeeded': False,
'error': 'noProject',
'errorMsg': 'Project does not exist'
}
# Check to make sure a project with the new name doesn't exist
newProjectPath = projectsPath + '/' + newProjectName
if os.path.isdir(newProjectPath):
# If not, create it
return {
'succeeded': False,
'error': 'projectExists',
'errorMsg': 'A project with the new name exists'
}
# Retrieve the project document
filePath = oldProjectPath + '/project.json'
project = cc.readJSONFile(filePath)
project['projectName'] = newProjectName
# Rename the project folder
os.rename(oldProjectPath, newProjectPath)
# Write the project file to the project directory
with open(newProjectPath + '/project.json', 'w') as projectFile:
projectFile.write(json.dumps(project, indent=2))
# Return the
return {'succeeded': True, 'project': project}
except Exception as err:
return {
'succeeded': False,
'error': 'errorUpdatingProject',
'errorMsg': 'Error updating project',
'detail': str(err)
}
def upload(resourceNames, data, uploadType='folder'):
# Get project name and folder name from resourceNames
projectName = resourceNames.get('projectname', None)
folderName = resourceNames.get('foldername', None)
fileName = resourceNames.get('filename', None)
# Check that project name and folder name are OK
if not (projectName and cc.legalName(projectName) and folderName
and cc.legalName(folderName)):
return {
'succeeded':
False,
'error':
'badProjectFolderName',
'errorMsg':
'Missing or invalid project or folder name',
'detail':
'Project name: ' + (projectName if projectName else '') + '\n' +
'Folder name: ' + (folderName if folderName else '')
}
# Check that project directory and project exist
projectPath = projectsPath + '/' + projectName
filePath = projectPath + '/project.json'
if not (os.path.isdir(projectPath) and os.path.isfile(filePath)):
return {
'succeeded': False,
'error': 'projectNotExist',
'errorMsg': 'Project does not exist'
}
# Get the project object from the project directory
project = cc.readJSONFile(filePath)
if not project:
return {
'succeeded': False,
'error': 'projectNotExist',
'errorMsg': 'Project does not exist'
}
# Get the data from the request
#data = request.get_data()
# Write out the files
newFileNames = []
fullFolderName = projectPath + '/' + folderName + '/'
uploadType = uploadType.lower()
try:
if uploadType == 'file':
newFileName = ci.writeFile(data, fullFolderName, fileName)
if newFileName:
newFileNames.append(newFileName)
elif uploadType == 'zipfile':
newFileNames = ci.writeZipFiles(data, fullFolderName)
else:
newFileNames = ci.writeFiles(data, fullFolderName)
except Exception as err:
return {
'succeeded': False,
'error': 'errorWritingFiles',
'errorMsg': 'Error writing files',
'detail': str(err)
}
# Get existing folder, if any, add if none
folders = project.get('folders', None)
if not folders:
folders = {}
project['folders'] = folders
folder = folders.get(folderName, None)
if not folder:
folder = {}
folders[folderName] = folder
# Get the list of new file names in the top level directory of the folder
newFileNames = ci.regexFilter(
[s[len(fullFolderName):] for s in newFileNames], '^[^/]*$')
# Add to the existing list if appropriate
fileNames = folder.get('fileNames', [])
if uploadType == 'file':
fileNames = fileNames.extend(newFileNames)
else:
fileNames = newFileNames
# Remove duplicates and update in the project
fileNames = list(set(fileNames))
folder['fileNames'] = fileNames
# Write the updated project to the project folder and return the project.
return {'succeeded': True, 'project': project}
import traceback
import re
import shutil
import docker
from flask import request
from werkzeug.utils import secure_filename
# Try to read the context
context = None
#dockerURL = None
#if os.path.exists('context.json'):
# context = cc.readJSONFile('context.json')
#if os.path.exists('/config/context.json'):
# context = cc.readJSONFile('/config/context.json')
if os.path.exists('/config/context.json'):
context = cc.readJSONFile('/config/context.json')
else:
# TODO: THIS IS AN ERROR!
pass
# Try to define the projects folder and working directory, create if necessary
if context:
projectsPath = '/projects'
workingDirectory = '/working'
#dockerURL = context.get('dockerURL', 'unix://var/run/docker.sock')
osPlatform = context.get('osPlatform', 'Darwin')
baseImage = context.get('baseImage', None)
if not os.path.isdir(projectsPath):
os.mkdir(projectsPath)
def test_walkNumber(self):
projectFilePath = os.path.abspath('../examples/project1/project.json')
project = cnc.readJSONFile(projectFilePath)
cnb.walkNumber(project)
self.assertTrue(project['folders']['pyfolder']['modules']['sum']['nodeNumber'])
