def __init__(self):
logging.basicConfig(
filename=self.LOGFILE,
format="%(levelname)s|%(asctime)s|%(name)s|%(message)s",
level=Config.LOGLEVEL)
self.log = logging.getLogger("TangoREST")
self.log.info("Starting RESTful Tango server")
self.tango = TangoServer()
self.status = Status()
def __init__(self):
logging.basicConfig(
filename = self.LOGFILE,
format = "%(levelname)s|%(asctime)s|%(name)s|%(message)s",
level = Config.LOGLEVEL
)
self.log = logging.getLogger("TangoREST")
self.log.info("Starting RESTful Tango server")
self.tango = TangoServer()
self.status = Status()
class TangoREST:
COURSELABS = Config.COURSELABS
OUTPUT_FOLDER = Config.OUTPUT_FOLDER
LOGFILE = Config.LOGFILE
# Replace with choice of key store and override validateKey.
# This key is just for testing.
KEYS = Config.KEYS
def __init__(self):
logging.basicConfig(
filename = self.LOGFILE,
format = "%(levelname)s|%(asctime)s|%(name)s|%(message)s",
level = Config.LOGLEVEL
)
self.log = logging.getLogger("TangoREST")
self.log.info("Starting RESTful Tango server")
self.tango = TangoServer()
self.status = Status()
def validateKey(self, key):
""" validateKey - Validates key provided by client
"""
result = False
for el in self.KEYS:
if el == key:
result = True
return result
def getDirName(self, key, courselab):
""" getDirName - Computes directory name
"""
return "%s-%s" % (key, courselab)
def getDirPath(self, key, courselab):
""" getDirPath - Computes directory path
"""
labName = self.getDirName(key, courselab)
return "%s/%s" % (self.COURSELABS, labName)
def getOutPath(self, key, courselab):
""" getOutPath - Computes output directory path
"""
labPath = self.getDirPath(key, courselab)
return "%s/%s" % (labPath, self.OUTPUT_FOLDER)
def getOutFilePath(self, key, courselab, outFile):
""" getOutFilePath - Compute output file name.
"""
outPath = self.getOutPath(key, courselab)
return "%s/%s" % (outPath, outFile)
def checkFileExists(self, directory, filename, fileMD5):
""" checkFileExists - Checks if a file exists in a
directory
"""
for elem in os.listdir(directory):
if elem == filename:
try:
body = open("%s/%s" % (directory, elem)).read()
md5hash = hashlib.md5(body).hexdigest()
return md5hash == fileMD5
except IOError:
continue
def createTangoMachine(self, image, vmms=Config.VMMS_NAME,
vmObj={'cores': 1, 'memory': 512, 'fallback_instance_type': 't2.nano'}):
""" createTangoMachine - Creates a tango machine object from image
"""
return TangoMachine(
name=image,
vmms=vmms,
image="%s" % (image),
cores=vmObj["cores"],
memory=vmObj["memory"],
fallback_instance_type=vmObj["fallback_instance_type"],
disk=None,
network=None)
def convertJobObj(self, key, courselab, jobObj):
""" convertJobObj - Converts a dictionary into a TangoJob object
"""
name = jobObj['jobName']
limitingKey = jobObj['limitingKey']
dirPath = self.getDirPath(key, courselab)
outputFile = self.getOutFilePath(key, courselab, jobObj['output_file'])
timeout = jobObj['timeout']
notifyURL = None
maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE
if 'callback_url' in jobObj:
notifyURL = jobObj['callback_url']
# List of input files
input = []
for file in jobObj['files']:
inFile = file['localFile']
vmFile = file['destFile']
handinfile = InputFile(
localFile="%s/%s" % (dirPath, inFile),
destFile=vmFile)
input.append(handinfile)
# VM object
if "vm" in jobObj:
vm = self.createTangoMachine(jobObj["image"], vmObj=jobObj["vm"])
else:
vm = self.createTangoMachine(jobObj["image"])
# for backward compatibility
accessKeyId = None
accessKey = None
if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0:
accessKeyId = jobObj["accessKeyId"]
accessKey = jobObj["accessKey"]
job = TangoJob(
name=name,
limitingKey=limitingKey,
vm=vm,
outputFile=outputFile,
input=input,
timeout=timeout,
notifyURL=notifyURL,
maxOutputFileSize=maxOutputFileSize,
accessKey=accessKey,
accessKeyId=accessKeyId
)
self.log.debug("inputFiles: %s" % [file.localFile for file in input])
self.log.debug("outputFile: %s" % outputFile)
return job
def convertTangoMachineObj(self, tangoMachine):
""" convertVMObj - Converts a TangoMachine object into a dictionary
"""
# May need to convert instance_id
vm = dict()
vm['network'] = tangoMachine.network
vm['resume'] = tangoMachine.resume
vm['image'] = tangoMachine.image
vm['memory'] = tangoMachine.memory
vm['vmms'] = tangoMachine.vmms
vm['cores'] = tangoMachine.cores
vm['fallback_instance_type'] = tangoMachine.fallback_instance_type
vm['disk'] = tangoMachine.disk
vm['id'] = tangoMachine.id
vm['name'] = tangoMachine.name
return vm
def convertInputFileObj(self, inputFile):
""" convertInputFileObj - Converts an InputFile object into a dictionary
"""
input = dict()
input['destFile'] = inputFile.destFile
input['localFile'] = inputFile.localFile
return input
def convertTangoJobObj(self, tangoJobObj):
""" convertTangoJobObj - Converts a TangoJob object into a dictionary
"""
job = dict()
# Convert scalar attribtues first
job['retries'] = tangoJobObj.retries
job['outputFile'] = tangoJobObj.outputFile
job['name'] = tangoJobObj.name
job['limitingKey'] = tangoJobObj.limitingKey
job['notifyURL'] = tangoJobObj.notifyURL
job['maxOutputFileSize'] = tangoJobObj.maxOutputFileSize
job['assigned'] = tangoJobObj.assigned
job['timeout'] = tangoJobObj.timeout
job['id'] = tangoJobObj.id
job['trace'] = tangoJobObj.trace
# Convert VM object
job['vm'] = self.convertTangoMachineObj(tangoJobObj.vm)
# Convert InputFile objects
inputFiles = list()
for inputFile in tangoJobObj.input:
inputFiles.append(self.convertInputFileObj(inputFile))
job['input'] = inputFiles
return job
##
# Tango RESTful API
##
def open(self, key, courselab):
""" open - Return a dict of md5 hashes for each input file in the
key-courselab directory and make one if the directory doesn't exist
"""
self.log.debug("Received open request(%s, %s)" % (key, courselab))
if self.validateKey(key):
labPath = self.getDirPath(key, courselab)
try:
if os.path.exists(labPath):
self.log.info(
"Found directory for (%s, %s)" % (key, courselab))
statusObj = self.status.found_dir
statusObj['files'] = {}
return statusObj
else:
outputPath = self.getOutPath(key, courselab)
os.makedirs(outputPath)
self.log.info(
"Created directory for (%s, %s)" % (key, courselab))
statusObj = self.status.made_dir
statusObj["files"] = {}
return statusObj
except Exception as e:
self.log.error("open request failed: %s" % str(e))
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def upload(self, key, courselab, file, body):
""" upload - Upload file as an input file in key-courselab if the
same file doesn't exist already
"""
self.log.debug("Received upload request(%s, %s, %s)" %
(key, courselab, file))
if (self.validateKey(key)):
labPath = self.getDirPath(key, courselab)
try:
if os.path.exists(labPath):
fileMD5 = hashlib.md5(body).hexdigest()
if self.checkFileExists(labPath, file, fileMD5):
self.log.info(
"File (%s, %s, %s) exists" % (key, courselab, file))
return self.status.file_exists
absPath = "%s/%s" % (labPath, file)
fh = open(absPath, "wt")
fh.write(body)
fh.close()
self.log.info(
"Uploaded file to (%s, %s, %s)" %
(key, courselab, file))
return self.status.file_uploaded
else:
self.log.info(
"Courselab for (%s, %s) not found" % (key, courselab))
return self.status.wrong_courselab
except Exception as e:
self.log.error("upload request failed: %s" % str(e))
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def addJob(self, key, courselab, jobStr):
""" addJob - Add the job to be processed by Tango
"""
self.log.debug("Received addJob request(%s, %s, %s)" %
(key, courselab, jobStr))
if (self.validateKey(key)):
try:
jobObj = json.loads(jobStr)
job = self.convertJobObj(key, courselab, jobObj)
jobId = self.tango.addJob(job)
self.log.debug("Done adding job")
if (jobId == -1):
self.log.info("Failed to add job to tango")
return self.status.create(-1, job.trace)
self.log.info("Successfully added job ID: %s to tango" % str(jobId))
result = self.status.job_added
result['jobId'] = jobId
return result
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
self.log.error("addJob request failed: %s" % str(e))
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def poll(self, key, courselab, outputFile, inprogress):
""" poll - Poll for the output file in key-courselab
"""
self.log.debug("Received poll request(%s, %s, %s, inprogress=%s)" %
(key, courselab, outputFile, inprogress))
if (self.validateKey(key)):
outFilePath = self.getOutFilePath(key, courselab, outputFile)
hdrfile = outFilePath + ".hdr"
bodyfile = outFilePath + ".body"
if inprogress and os.path.exists(hdrfile) and os.path.exists(bodyfile):
self.log.info("In-progress output files (%s, %s, %s, %s) found" %
(key, courselab, hdrfile, bodyfile))
runningTime = self.tango.runningTimeForOutputFile(outFilePath)
if runningTime:
result = "Total running time: %s seconds\n\n" % runningTime.seconds
else:
result = ""
output = open(hdrfile)
result += output.read()
output.close()
result += "In-progress autodriver output from grading VM:\n\n"
output = open(bodyfile)
result += output.read()
output.close()
return result
if not inprogress and os.path.exists(outFilePath):
self.log.info("Output file (%s, %s, %s) found" %
(key, courselab, outputFile))
output = open(outFilePath)
result = output.read()
output.close()
return result
self.log.info("Output file (%s, %s, %s) not found" %
(key, courselab, outputFile))
return self.status.out_not_found
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def info(self, key):
""" info - Returns basic status for the Tango service such as uptime, number of jobs etc
"""
self.log.debug("Received info request (%s)" % (key))
if (self.validateKey(key)):
info = self.tango.getInfo()
result = self.status.obtained_info
result['info'] = info
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def jobs(self, key, deadJobs):
""" jobs - Returns the list of live jobs (deadJobs == 0) or the list of dead jobs (deadJobs == 1)
"""
self.log.debug("Received jobs request (%s, %s)" % (key, deadJobs))
if (self.validateKey(key)):
jobs = list()
result = self.status.obtained_jobs
if (int(deadJobs) == 0):
jobs = self.tango.getJobs(0)
self.log.debug(
"Retrieved %d live jobs (deadJobs = %s)" % (len(jobs), deadJobs))
elif (int(deadJobs) == 1):
jobs = self.tango.getJobs(-1)
self.log.debug(
"Retrieved %d dead jobs (deadJobs = %s)" % (len(jobs), deadJobs))
result['jobs'] = list()
for job in jobs:
result['jobs'].append(self.convertTangoJobObj(job))
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def pool(self, key, image):
""" pool - Get information about pool(s) of VMs
"""
self.log.debug("Received pool request(%s, %s)" % (key, image))
if self.validateKey(key):
pools = self.tango.preallocator.getAllPools()
self.log.info("All pools found")
if image == "":
result = self.status.obtained_all_pools
else:
if image in pools:
pools = {image: pools[image]}
self.log.info("Pool image found: %s" % image)
result = self.status.obtained_pool
else:
self.log.info("Invalid image name: %s" % image)
result = self.status.pool_not_found
result["pools"] = pools
result["low_water_mark"] = TangoIntValue("low_water_mark", -1).get()
result["max_pool_size"] = TangoIntValue("max_pool_size", -1).get()
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def prealloc(self, key, image, num, vmStr):
""" prealloc - Create a pool of num instances spawned from image
"""
self.log.debug("Received prealloc request(%s, %s, %s)" %
(key, image, num))
if self.validateKey(key):
if vmStr != "":
vmObj = json.loads(vmStr)
vm = self.createTangoMachine(image, vmObj=vmObj)
else:
vm = self.createTangoMachine(image)
ret = self.tango.preallocVM(vm, int(num))
if ret == -1:
self.log.error("Prealloc failed")
return self.status.prealloc_failed
if ret == -2:
self.log.error("Invalid prealloc size")
return self.status.invalid_prealloc_size
if ret == -3:
self.log.error("Invalid image name")
return self.status.invalid_image
self.log.info("Successfully preallocated VMs")
return self.status.preallocated
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def scale(self, key, low_water_mark, max_pool_size):
""" scale - Set the low water mark and max pool size parameters to control auto-scaling
"""
self.log.debug("Received scale request(%s, %s)" % (low_water_mark, max_pool_size))
if self.validateKey(key):
ret = self.tango.setScaleParams(low_water_mark, max_pool_size)
if ret == -1:
self.log.error("Scale failed")
return self.status.scale_failed
self.log.info("Successfully updated scale params")
return self.status.scale_updated
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def cancel(self, key, courselab, outputFile):
""" cancel - Cancel the job with output file outputfile.
"""
self.log.debug("Received cancel request(%s, %s)" % (key, outputFile))
if self.validateKey(key):
outFilePath = self.getOutFilePath(key, courselab, outputFile)
ret = self.tango.cancelJobWithPath(outFilePath)
if ret == CancellationStatus.NOT_FOUND:
self.log.error("No job was found with output file %s, so no job was cancelled." % outputFile)
return self.status.job_cancellation_failed_not_found
elif ret == CancellationStatus.ALREADY_COMPLETED:
self.log.error("The job found with output file %s had already completed, so no job was cancelled." % outputFile)
return self.status.job_cancellation_spurious
elif ret == CancellationStatus.FAILED:
self.log.error("The job found with output file %s could not be cancelled" % outputFile)
return self.status.job_cancellation_failed
self.log.info("Successfully registered request to cancel job with output file %s" % outputFile)
return self.status.job_cancelled
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
class TangoREST:
COURSELABS = Config.COURSELABS
OUTPUT_FOLDER = Config.OUTPUT_FOLDER
LOGFILE = Config.LOGFILE
# Replace with choice of key store and override validateKey.
# This key is just for testing.
KEYS = Config.KEYS
def __init__(self):
logging.basicConfig(
filename = self.LOGFILE,
format = "%(levelname)s|%(asctime)s|%(name)s|%(message)s",
level = Config.LOGLEVEL
)
self.log = logging.getLogger("TangoREST")
self.log.info("Starting RESTful Tango server")
self.tango = TangoServer()
self.status = Status()
def validateKey(self, key):
""" validateKey - Validates key provided by client
"""
result = False
for el in self.KEYS:
if el == key:
result = True
return result
def getDirName(self, key, courselab):
""" getDirName - Computes directory name
"""
return "%s-%s" % (key, courselab)
def getDirPath(self, key, courselab):
""" getDirPath - Computes directory path
"""
labName = self.getDirName(key, courselab)
return "%s/%s" % (self.COURSELABS, labName)
def getOutPath(self, key, courselab):
""" getOutPath - Computes output directory path
"""
labPath = self.getDirPath(key, courselab)
return "%s/%s" % (labPath, self.OUTPUT_FOLDER)
def checkFileExists(self, directory, filename, fileMD5):
""" checkFileExists - Checks if a file exists in a
directory
"""
for elem in os.listdir(directory):
if elem == filename:
try:
body = open("%s/%s" % (directory, elem)).read()
md5hash = hashlib.md5(body).hexdigest()
return md5hash == fileMD5
except IOError:
continue
def createTangoMachine(self, image, vmms=Config.VMMS_NAME,
vmObj={'cores': 1, 'memory': 512}):
""" createTangoMachine - Creates a tango machine object from image
"""
return TangoMachine(
name=image,
vmms=vmms,
image="%s" % (image),
cores=vmObj["cores"],
memory=vmObj["memory"],
disk=None,
network=None)
def convertJobObj(self, dirName, jobObj):
""" convertJobObj - Converts a dictionary into a TangoJob object
"""
name = jobObj['jobName']
outputFile = "%s/%s/%s/%s" % (self.COURSELABS,
dirName,
self.OUTPUT_FOLDER,
jobObj['output_file'])
timeout = jobObj['timeout']
notifyURL = None
maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE
if 'callback_url' in jobObj:
notifyURL = jobObj['callback_url']
# List of input files
input = []
for file in jobObj['files']:
inFile = file['localFile']
vmFile = file['destFile']
handinfile = InputFile(
localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile),
destFile=vmFile)
input.append(handinfile)
# VM object
vm = self.createTangoMachine(jobObj["image"])
job = TangoJob(
name=name,
vm=vm,
outputFile=outputFile,
input=input,
timeout=timeout,
notifyURL=notifyURL,
maxOutputFileSize=maxOutputFileSize)
self.log.debug("inputFiles: %s" % [file.localFile for file in input])
self.log.debug("outputFile: %s" % outputFile)
return job
def convertTangoMachineObj(self, tangoMachine):
""" convertVMObj - Converts a TangoMachine object into a dictionary
"""
# May need to convert instance_id
vm = dict()
vm['network'] = tangoMachine.network
vm['resume'] = tangoMachine.resume
vm['image'] = tangoMachine.image
vm['memory'] = tangoMachine.memory
vm['vmms'] = tangoMachine.vmms
vm['cores'] = tangoMachine.cores
vm['disk'] = tangoMachine.disk
vm['id'] = tangoMachine.id
vm['name'] = tangoMachine.name
return vm
def convertInputFileObj(self, inputFile):
""" convertInputFileObj - Converts an InputFile object into a dictionary
"""
input = dict()
input['destFile'] = inputFile.destFile
input['localFile'] = inputFile.localFile
return input
def convertTangoJobObj(self, tangoJobObj):
""" convertTangoJobObj - Converts a TangoJob object into a dictionary
"""
job = dict()
# Convert scalar attribtues first
job['retries'] = tangoJobObj.retries
job['outputFile'] = tangoJobObj.outputFile
job['name'] = tangoJobObj.name
job['notifyURL'] = tangoJobObj.notifyURL
job['maxOutputFileSize'] = tangoJobObj.maxOutputFileSize
job['assigned'] = tangoJobObj.assigned
job['timeout'] = tangoJobObj.timeout
job['id'] = tangoJobObj.id
job['trace'] = tangoJobObj.trace
# Convert VM object
job['vm'] = self.convertTangoMachineObj(tangoJobObj.vm)
# Convert InputFile objects
inputFiles = list()
for inputFile in tangoJobObj.input:
inputFiles.append(self.convertInputFileObj(inputFile))
job['input'] = inputFiles
return job
##
# Tango RESTful API
##
def open(self, key, courselab):
""" open - Return a dict of md5 hashes for each input file in the
key-courselab directory and make one if the directory doesn't exist
"""
self.log.debug("Received open request(%s, %s)" % (key, courselab))
if self.validateKey(key):
labPath = self.getDirPath(key, courselab)
try:
if os.path.exists(labPath):
self.log.info(
"Found directory for (%s, %s)" % (key, courselab))
statusObj = self.status.found_dir
statusObj['files'] = {}
return statusObj
else:
outputPath = self.getOutPath(key, courselab)
os.makedirs(outputPath)
self.log.info(
"Created directory for (%s, %s)" % (key, courselab))
statusObj = self.status.made_dir
statusObj["files"] = {}
return statusObj
except Exception as e:
self.log.error("open request failed: %s" % str(e))
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def upload(self, key, courselab, file, body):
""" upload - Upload file as an input file in key-courselab if the
same file doesn't exist already
"""
self.log.debug("Received upload request(%s, %s, %s)" %
(key, courselab, file))
if (self.validateKey(key)):
labPath = self.getDirPath(key, courselab)
try:
if os.path.exists(labPath):
fileMD5 = hashlib.md5(body).hexdigest()
if self.checkFileExists(labPath, file, fileMD5):
self.log.info(
"File (%s, %s, %s) exists" % (key, courselab, file))
return self.status.file_exists
absPath = "%s/%s" % (labPath, file)
fh = open(absPath, "wt")
fh.write(body)
fh.close()
self.log.info(
"Uploaded file to (%s, %s, %s)" %
(key, courselab, file))
return self.status.file_uploaded
else:
self.log.info(
"Courselab for (%s, %s) not found" % (key, courselab))
return self.status.wrong_courselab
except Exception as e:
self.log.error("upload request failed: %s" % str(e))
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def addJob(self, key, courselab, jobStr):
""" addJob - Add the job to be processed by Tango
"""
self.log.debug("Received addJob request(%s, %s, %s)" %
(key, courselab, jobStr))
if (self.validateKey(key)):
labName = self.getDirName(key, courselab)
try:
jobObj = json.loads(jobStr)
job = self.convertJobObj(labName, jobObj)
jobId = self.tango.addJob(job)
self.log.debug("Done adding job")
if (jobId == -1):
self.log.info("Failed to add job to tango")
return self.status.create(-1, job.trace)
self.log.info("Successfully added job ID: %s to tango" % str(jobId))
result = self.status.job_added
result['jobId'] = jobId
return result
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
self.log.error("addJob request failed: %s" % str(e))
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def poll(self, key, courselab, outputFile):
""" poll - Poll for the output file in key-courselab
"""
self.log.debug("Received poll request(%s, %s, %s)" %
(key, courselab, outputFile))
if (self.validateKey(key)):
outputPath = self.getOutPath(key, courselab)
outfilePath = "%s/%s" % (outputPath, outputFile)
if os.path.exists(outfilePath):
self.log.info("Output file (%s, %s, %s) found" %
(key, courselab, outputFile))
output = open(outfilePath)
result = output.read()
output.close()
return result
self.log.info("Output file (%s, %s, %s) not found" %
(key, courselab, outputFile))
return self.status.out_not_found
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def info(self, key):
""" info - Returns basic status for the Tango service such as uptime, number of jobs etc
"""
self.log.debug("Received info request (%s)" % (key))
if (self.validateKey(key)):
info = self.tango.getInfo()
result = self.status.obtained_info
result['info'] = info
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def jobs(self, key, deadJobs):
""" jobs - Returns the list of live jobs (deadJobs == 0) or the list of dead jobs (deadJobs == 1)
"""
self.log.debug("Received jobs request (%s, %s)" % (key, deadJobs))
if (self.validateKey(key)):
jobs = list()
result = self.status.obtained_jobs
if (int(deadJobs) == 0):
jobs = self.tango.getJobs(0)
self.log.debug(
"Retrieved live jobs (deadJobs = %s)" % deadJobs)
elif (int(deadJobs) == 1):
jobs = self.tango.getJobs(-1)
self.log.debug(
"Retrieved dead jobs (deadJobs = %s)" % deadJobs)
result['jobs'] = list()
for job in jobs:
result['jobs'].append(self.convertTangoJobObj(job))
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def pool(self, key, image):
""" pool - Get information about a pool of VMs spawned from image
"""
self.log.debug("Received pool request(%s, %s)" % (key, image))
if self.validateKey(key):
if image == "":
pools = self.tango.preallocator.getAllPools()
else:
info = self.tango.preallocator.getPool(image)
pools = {}
if len(info) > 0:
pools[image] = info
if len(pools) > 0:
self.log.info("Pool image found: %s" % image)
result = self.status.obtained_pool
else:
self.log.info("Invalid image name: %s" % image)
result = self.status.pool_not_found
result["pools"] = pools
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def prealloc(self, key, image, num, vmStr):
""" prealloc - Create a pool of num instances spawned from image
"""
self.log.debug("Received prealloc request(%s, %s, %s)" %
(key, image, num))
if self.validateKey(key):
if vmStr != "":
vmObj = json.loads(vmStr)
vm = self.createTangoMachine(image, vmObj=vmObj)
else:
vm = self.createTangoMachine(image)
ret = self.tango.preallocVM(vm, int(num))
if ret == -1:
self.log.error("Prealloc failed")
return self.status.prealloc_failed
if ret == -2:
self.log.error("Invalid prealloc size")
return self.status.invalid_prealloc_size
if ret == -3:
self.log.error("Invalid image name")
return self.status.invalid_image
self.log.info("Successfully preallocated VMs")
return self.status.preallocated
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
print "after allocation", key, total, free
# When a host has two Tango containers (for experiment), there are two
# redis servers, too. They differ by the forwarding port number, which
# is defined in config_for_run_jobs.py. To select the redis server,
# We get the connection here and pass it into tangoObjects
redisConnection = redis.StrictRedis(
host=Config.REDIS_HOSTNAME,
port=config_for_run_jobs.Config.redisHostPort,
db=0)
tangoObjects.getRedisConnection(connection=redisConnection)
boto3connection = boto3.client("ec2", Config.EC2_REGION)
boto3resource = boto3.resource("ec2", Config.EC2_REGION)
server = TangoServer()
ec2 = server.preallocator.vmms["ec2SSH"]
pools = ec2.img2ami
if destroyList:
print "Current"
listInstances()
totalTerminated = []
for partialStr in destroyList:
matchingInstances = []
if partialStr.lower() == "NoNameTag".lower(): # without "Name" tag
for item in sortedInstances:
if "None" == instanceNameTag(item["Instance"]):
matchingInstances.append(item)
elif partialStr.startswith("i-"): # match instance id
class TangoREST(object):
COURSELABS = Config.COURSELABS
OUTPUT_FOLDER = Config.OUTPUT_FOLDER
LOGFILE = Config.LOGFILE
# Replace with choice of key store and override validateKey.
# This key is just for testing.
KEYS = Config.KEYS
def __init__(self):
logging.basicConfig(
filename=self.LOGFILE,
format="%(levelname)s|%(asctime)s|%(name)s|%(message)s",
level=Config.LOGLEVEL,
)
self.log = logging.getLogger("TangoREST")
self.log.info("Starting RESTful Tango server")
self.tango = TangoServer()
self.status = Status()
def validateKey(self, key):
"""validateKey - Validates key provided by client"""
result = False
for el in self.KEYS:
if el == key:
result = True
return result
def getDirName(self, key, courselab):
"""getDirName - Computes directory name"""
return "%s-%s" % (key, courselab)
def getDirPath(self, key, courselab):
"""getDirPath - Computes directory path"""
labName = self.getDirName(key, courselab)
return "%s/%s" % (self.COURSELABS, labName)
def getOutPath(self, key, courselab):
"""getOutPath - Computes output directory path"""
labPath = self.getDirPath(key, courselab)
return "%s/%s" % (labPath, self.OUTPUT_FOLDER)
def checkFileExists(self, directory, filename, fileMD5):
"""checkFileExists - Checks if a file exists in a
directory
"""
for elem in os.listdir(directory):
if elem == filename:
try:
body = open("%s/%s" % (directory, elem), "rb").read()
md5hash = hashlib.md5(body).hexdigest()
return md5hash == fileMD5
except IOError:
continue
def createTangoMachine(self,
image,
vmms=Config.VMMS_NAME,
vmObj={
"cores": 1,
"memory": 512
}):
"""createTangoMachine - Creates a tango machine object from image"""
return TangoMachine(
name=image,
vmms=vmms,
image="%s" % (image),
cores=vmObj["cores"],
memory=vmObj["memory"],
disk=None,
network=None,
)
def convertJobObj(self, dirName, jobObj):
"""convertJobObj - Converts a dictionary into a TangoJob object"""
name = jobObj["jobName"]
outputFile = "%s/%s/%s/%s" % (
self.COURSELABS,
dirName,
self.OUTPUT_FOLDER,
jobObj["output_file"],
)
timeout = jobObj["timeout"]
notifyURL = None
maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE
if "callback_url" in jobObj:
notifyURL = jobObj["callback_url"]
# List of input files
input = []
for file in jobObj["files"]:
inFile = file["localFile"]
vmFile = file["destFile"]
handinfile = InputFile(
localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile),
destFile=vmFile,
)
input.append(handinfile)
# VM object
vm = self.createTangoMachine(jobObj["image"])
# for backward compatibility
accessKeyId = None
accessKey = None
if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0:
accessKeyId = jobObj["accessKeyId"]
accessKey = jobObj["accessKey"]
job = TangoJob(
name=name,
vm=vm,
outputFile=outputFile,
input=input,
timeout=timeout,
notifyURL=notifyURL,
maxOutputFileSize=maxOutputFileSize,
accessKey=accessKey,
accessKeyId=accessKeyId,
)
self.log.debug("inputFiles: %s" % [file.localFile for file in input])
self.log.debug("outputFile: %s" % outputFile)
return job
def convertTangoMachineObj(self, tangoMachine):
"""convertVMObj - Converts a TangoMachine object into a dictionary"""
# May need to convert instance_id
vm = dict()
vm["network"] = tangoMachine.network
vm["resume"] = tangoMachine.resume
vm["image"] = tangoMachine.image
vm["memory"] = tangoMachine.memory
vm["vmms"] = tangoMachine.vmms
vm["cores"] = tangoMachine.cores
vm["disk"] = tangoMachine.disk
vm["id"] = tangoMachine.id
vm["name"] = tangoMachine.name
return vm
def convertInputFileObj(self, inputFile):
"""convertInputFileObj - Converts an InputFile object into a dictionary"""
input = dict()
input["destFile"] = inputFile.destFile
input["localFile"] = inputFile.localFile
return input
def convertTangoJobObj(self, tangoJobObj):
"""convertTangoJobObj - Converts a TangoJob object into a dictionary"""
job = dict()
# Convert scalar attribtues first
job["retries"] = tangoJobObj.retries
job["outputFile"] = tangoJobObj.outputFile
job["name"] = tangoJobObj.name
job["notifyURL"] = tangoJobObj.notifyURL
job["maxOutputFileSize"] = tangoJobObj.maxOutputFileSize
job["assigned"] = tangoJobObj.assigned
job["timeout"] = tangoJobObj.timeout
job["id"] = tangoJobObj.id
job["trace"] = tangoJobObj.trace
# Convert VM object
job["vm"] = self.convertTangoMachineObj(tangoJobObj.vm)
# Convert InputFile objects
inputFiles = list()
for inputFile in tangoJobObj.input:
inputFiles.append(self.convertInputFileObj(inputFile))
job["input"] = inputFiles
return job
##
# Tango RESTful API
##
def open(self, key, courselab):
"""open - Return a dict of md5 hashes for each input file in the
key-courselab directory and make one if the directory doesn't exist
"""
self.log.debug("Received open request(%s, %s)" % (key, courselab))
if self.validateKey(key):
labPath = self.getDirPath(key, courselab)
try:
if os.path.exists(labPath):
self.log.info("Found directory for (%s, %s)" %
(key, courselab))
statusObj = self.status.found_dir
statusObj["files"] = {}
return statusObj
else:
outputPath = self.getOutPath(key, courselab)
os.makedirs(outputPath)
self.log.info("Created directory for (%s, %s)" %
(key, courselab))
statusObj = self.status.made_dir
statusObj["files"] = {}
return statusObj
except Exception as e:
self.log.error("open request failed: %s" % str(e))
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def upload(self, key, courselab, file, tempfile, fileMD5):
"""upload - Upload file as an input file in key-courselab if the
same file doesn't exist already
"""
self.log.debug("Received upload request(%s, %s, %s)" %
(key, courselab, file))
if self.validateKey(key):
labPath = self.getDirPath(key, courselab)
try:
if os.path.exists(labPath):
if self.checkFileExists(labPath, file, fileMD5):
self.log.info("File (%s, %s, %s) exists" %
(key, courselab, file))
os.unlink(tempfile)
return self.status.file_exists
absPath = "%s/%s" % (labPath, file)
os.rename(tempfile, absPath)
self.log.info("Uploaded file to (%s, %s, %s)" %
(key, courselab, file))
return self.status.file_uploaded
else:
self.log.info("Courselab for (%s, %s) not found" %
(key, courselab))
os.unlink(tempfile)
return self.status.wrong_courselab
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
self.log.error("upload request failed: %s" % str(e))
os.unlink(tempfile)
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
os.unlink(tempfile)
return self.status.wrong_key
def addJob(self, key, courselab, jobStr):
"""addJob - Add the job to be processed by Tango"""
self.log.debug("Received addJob request(%s, %s, %s)" %
(key, courselab, jobStr))
if self.validateKey(key):
labName = self.getDirName(key, courselab)
try:
jobObj = json.loads(jobStr)
job = self.convertJobObj(labName, jobObj)
jobId = self.tango.addJob(job)
self.log.debug("Done adding job")
if jobId == -1:
self.log.info("Failed to add job to tango")
return self.status.create(-1, job.trace)
self.log.info("Successfully added job ID: %s to tango" %
str(jobId))
result = self.status.job_added
result["jobId"] = jobId
return result
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
self.log.error("addJob request failed: %s" % str(e))
return self.status.create(-1, str(e))
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def poll(self, key, courselab, outputFile):
"""poll - Poll for the output file in key-courselab"""
self.log.debug("Received poll request(%s, %s, %s)" %
(key, courselab, outputFile))
if self.validateKey(key):
outputPath = self.getOutPath(key, courselab)
outfilePath = "%s/%s" % (outputPath, outputFile)
if os.path.exists(outfilePath):
self.log.info("Output file (%s, %s, %s) found" %
(key, courselab, outputFile))
output = open(outfilePath)
result = output.read()
output.close()
return result
self.log.info("Output file (%s, %s, %s) not found" %
(key, courselab, outputFile))
return self.status.out_not_found
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def info(self, key):
"""info - Returns basic status for the Tango service such as uptime, number of jobs etc"""
self.log.debug("Received info request (%s)" % (key))
if self.validateKey(key):
info = self.tango.getInfo()
result = self.status.obtained_info
result["info"] = info
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def jobs(self, key, deadJobs):
"""jobs - Returns the list of live jobs (deadJobs == 0) or the list of dead jobs (deadJobs == 1)"""
self.log.debug("Received jobs request (%s, %s)" % (key, deadJobs))
if self.validateKey(key):
jobs = list()
result = self.status.obtained_jobs
if int(deadJobs) == 0:
jobs = self.tango.getJobs(0)
self.log.debug("Retrieved live jobs (deadJobs = %s)" %
deadJobs)
elif int(deadJobs) == 1:
jobs = self.tango.getJobs(-1)
self.log.debug("Retrieved dead jobs (deadJobs = %s)" %
deadJobs)
result["jobs"] = list()
for job in jobs:
result["jobs"].append(self.convertTangoJobObj(job))
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def pool(self, key, image):
"""pool - Get information about pool(s) of VMs"""
self.log.debug("Received pool request(%s, %s)" % (key, image))
if self.validateKey(key):
pools = self.tango.preallocator.getAllPools()
self.log.info("All pools found")
if image == "":
result = self.status.obtained_all_pools
else:
if image in pools:
pools = {image: pools[image]}
self.log.info("Pool image found: %s" % image)
result = self.status.obtained_pool
else:
self.log.info("Invalid image name: %s" % image)
result = self.status.pool_not_found
result["pools"] = pools
return result
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key
def prealloc(self, key, image, num, vmStr):
"""prealloc - Create a pool of num instances spawned from image"""
self.log.debug("Received prealloc request(%s, %s, %s)" %
(key, image, num))
if self.validateKey(key):
if vmStr != "":
vmObj = json.loads(vmStr)
vm = self.createTangoMachine(image, vmObj=vmObj)
else:
vm = self.createTangoMachine(image)
ret = self.tango.preallocVM(vm, int(num))
if ret == -1:
self.log.error("Prealloc failed")
return self.status.prealloc_failed
if ret == -2:
self.log.error("Invalid prealloc size")
return self.status.invalid_prealloc_size
if ret == -3:
self.log.error("Invalid image name")
return self.status.invalid_image
self.log.info("Successfully preallocated VMs")
return self.status.preallocated
else:
self.log.info("Key not recognized: %s" % key)
return self.status.wrong_key