def get_node_info(self, kwargs):
"""
HTTP GET method. Gets info of a specific node.
:param param: serviceNodeId is a VMID of an existing service node.
:type param: str
:returns: HttpJsonResponse - JSON response with details about the node.
:raises: ManagerException
"""
if 'serviceNodeId' not in kwargs:
return HttpErrorResponse(ManagerException(E_ARGS_MISSING, 'serviceNodeId').message)
serviceNodeId = kwargs.pop('serviceNodeId')
if len(kwargs) != 0:
return HttpErrorResponse(ManagerException(E_ARGS_UNEXPECTED, kwargs.keys()).message)
if serviceNodeId not in self.config.serviceNodes:
return HttpErrorResponse(ManagerException(E_ARGS_INVALID , \
"serviceNodeId" ,\
detail='Invalid "serviceNodeId"').message)
serviceNode = self.config.getMySQLNode(serviceNodeId)
return HttpJsonResponse({
'serviceNode': {
'id': serviceNode.id,
'ip': serviceNode.ip,
'isMaster': serviceNode.isMaster,
'isSlave': serviceNode.isSlave
}
})
def _update(self, post_params, class_file, pClass):
try:
if type(post_params) != dict: raise TypeError()
fd = open(class_file, 'r')
p = pickle.load(fd)
fd.close()
p.configure(**post_params)
p.restart()
except (ValueError, TypeError) as e:
self.logger.exception(e)
ex = AgentException(AgentException.E_ARGS_INVALID)
return HttpErrorResponse(ex.message)
except Exception as e:
self.logger.exception(e)
ex = AgentException(AgentException.E_UNKNOWN, detail=e)
return HttpErrorResponse(ex.message)
else:
try:
fd = open(class_file, 'w')
pickle.dump(p, fd)
fd.close()
except Exception as e:
self.logger.exception(ex.message)
ex = AgentException(AgentException.E_CONFIG_COMMIT_FAILED,
detail=e)
return HttpErrorResponse(ex.message)
else:
return HttpJsonResponse()
def get_node_info(self, kwargs):
"""Return information about the node identified by the given
kwargs['serviceNodeId']"""
# serviceNodeId is a required parameter
if 'serviceNodeId' not in kwargs:
vals = {'arg': 'serviceNodeId'}
return HttpErrorResponse(self.REQUIRED_ARG_MSG % vals)
serviceNodeId = kwargs.pop('serviceNodeId')
serviceNode = None
for node in self.nodes:
if serviceNodeId == node.id:
serviceNode = node
break
if serviceNode is None:
return HttpErrorResponse(
'ERROR: Cannot find node with serviceNode=%s' % serviceNodeId)
return HttpJsonResponse({
'serviceNode': {
'id': serviceNode.id,
'ip': serviceNode.ip,
'is_hub': self.__is_hub(serviceNode)
}
})
def _stop(self, kwargs, class_file, pClass):
self.logger.debug("_stop(kwargs=%s, class_file=%s, pClass=%s)" %
(kwargs, class_file, pClass))
if not exists(class_file):
self.logger.error("class_file '%s' does not exist" % class_file)
return HttpErrorResponse(
AgentException(AgentException.E_CONFIG_NOT_EXIST).message)
try:
try:
fd = open(class_file, 'r')
p = pickle.load(fd)
self.logger.debug('dump file %s loaded' % class_file)
fd.close()
except Exception as e:
ex = AgentException(AgentException.E_CONFIG_READ_FAILED,
detail=e)
self.logger.exception(ex.message)
return HttpErrorResponse(ex.message)
if 'drain' in kwargs:
p.stop(kwargs['drain'])
else:
p.stop()
self.logger.debug("Removing class_file '%s'" % class_file)
remove(class_file)
return HttpJsonResponse()
except Exception as e:
ex = AgentException(AgentException.E_UNKNOWN, detail=e)
self.logger.exception(e)
return HttpErrorResponse(ex.message)
def set_striping_policy(self, kwargs):
if self.state != self.S_RUNNING:
return HttpErrorResponse(
'ERROR: Wrong state to set Striping policy.')
if not 'volumeName' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (volumeName) doesn\'t exist')
if not 'policy' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (policy) doesn\'t exist')
if not 'width' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (factor) doesn\'t exist')
if not 'stripe-size' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (stripe-size) doesn\'t exist')
volumeName = kwargs.pop('volumeName')
policy = kwargs.pop('policy')
width = kwargs.pop('width')
stripe_size = kwargs.pop('stripe-size')
# xtfsutil <path> --set-dsp --striping-policy <policy> --striping-policy-width <width> --striping-policy-stripe-size <stripe-size>
args = [
'--set-dsp', '--striping-policy', policy,
'--striping-policy-width', width, '--striping-policy-stripe-size',
stripe_size
]
return self.set_policy(volumeName, 'Striping', args)
def set_password(self, kwargs):
self.logger.debug('Setting password')
if self.state != self.S_RUNNING:
self.logger.debug('Service not runnning')
return HttpErrorResponse('ERROR: Service not running')
if not 'user' in kwargs:
return HttpErrorResponse('ERROR: Required argument \'user\' doesn\'t exist')
if not 'password' in kwargs:
return HttpErrorResponse('ERROR: Required argument \'password\' doesn\'t exist')
# Get the master
masters = self.config.getMySQLmasters()
#TODO: modify this when multiple masters
try:
for master in masters:
client.set_password(master.ip, self.config.AGENT_PORT, kwargs['user'], kwargs['password'])
except:
e = sys.exc_info()[1]
self.logger.exception('set_password: Could not set password: %s' % e)
self.state = self.S_ERROR
return HttpErrorResponse('Failed to set password')
else:
self.root_pass = kwargs['password']
return HttpJsonResponse()
def set_replication_policy(self, kwargs):
if self.state != self.S_RUNNING:
return HttpErrorResponse(
'ERROR: Wrong state to set Replication policy.')
if not 'volumeName' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (volumeName) doesn\'t exist')
if not 'policy' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (policy) doesn\'t exist')
if not 'factor' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (factor) doesn\'t exist')
volumeName = kwargs.pop('volumeName')
policy = kwargs.pop('policy')
factor = kwargs.pop('factor')
# xtfsutil <path> --set-drp --replication-policy <policy> --replication-factor <factor>
args = [
'--set-drp', '--replication-policy', policy,
'--replication-factor', factor
]
return self.set_policy(volumeName, 'Replication', args)
def _create(self, post_params, class_file, pClass):
if exists(class_file):
return HttpErrorResponse(
AgentException(AgentException.E_CONFIG_EXISTS).message)
try:
if type(post_params) != dict:
raise TypeError()
self.logger.debug('Creating class')
p = pClass(**post_params)
self.logger.debug('Created class')
except (ValueError, TypeError) as e:
ex = AgentException(AgentException.E_ARGS_INVALID, detail=str(e))
self.logger.exception(e)
return HttpErrorResponse(ex.message)
except Exception as e:
ex = AgentException(AgentException.E_UNKNOWN, detail=e)
self.logger.exception(e)
return HttpErrorResponse(ex.message)
else:
try:
self.logger.debug('Openning file %s' % class_file)
fd = open(class_file, 'w')
pickle.dump(p, fd)
fd.close()
except Exception as e:
ex = AgentException(AgentException.E_CONFIG_COMMIT_FAILED,
detail=e)
self.logger.exception(ex.message)
return HttpErrorResponse(ex.message)
else:
self.logger.debug('Created class file')
return HttpJsonResponse()
def createVolume(self, kwargs):
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to create Volume')
if not 'volumeName' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (volumeName) doesn\'t exist')
volumeName = kwargs.pop('volumeName')
# Get the value of 'owner', if specified. 'xtreemfs' otherwise
owner = kwargs.pop('owner', 'xtreemfs')
args = [
'mkfs.xtreemfs',
'%s:32636/%s' % (self.mrcNodes[0].ip, volumeName), "-u", owner,
"-g", owner, "-m", "777"
]
process = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = process.communicate()
process.poll()
if process.returncode != 0:
self.logger.info('Failed to create volume: %s; %s', stdout, stderr)
return HttpErrorResponse("The volume could not be created")
self.logger.info('Creating Volume: %s; %s', stdout, stderr)
return HttpJsonResponse()
def startup(self, kwargs):
self.logger.info("HTC Manager starting up")
self.logger.info(str(kwargs))
if self.state != self.S_INIT and self.state != self.S_STOPPED:
vals = { 'curstate': self.state, 'action': 'startup' }
return HttpErrorResponse(self.WRONG_STATE_MSG % vals)
if 'cloud' in kwargs:
try:
self._init_cloud(kwargs['cloud'])
except Exception:
return HttpErrorResponse(
"A cloud named '%s' could not be found" % kwargs['cloud'])
#self.logger.info('Get service TaskFarm')
try:
self.service = TaskFarm(kwargs['mode'], kwargs['type'])
except (UnknownHtcTypeError, UnimplementedHtcCombinationError) as e:
return HttpErrorResponse({ 'error': e.__str__() })
#self.logger.info('Got service TaskFarm, delete some kwargs entries')
self.state = self.S_PROLOGUE
del kwargs['type']
del kwargs['mode']
#del kwargs['m_type']
#self.logger.info('Show leftover kwargs entries')
self.logger.info(str(kwargs))
#self.logger.info('Starting Thread for startup')
Thread(target=self._do_startup, kwargs=kwargs).start()
self.logger.info(str(self.service))
return HttpJsonResponse({ 'state': self.state })
def download_code_version(self, kwargs):
if 'codeVersionId' not in kwargs:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_MISSING,
'codeVersionId').message)
if isinstance(kwargs['codeVersionId'], dict):
return HttpErrorResponse(
ManagerException(
ManagerException.E_ARGS_INVALID,
detail='codeVersionId should be a string').message)
codeVersion = kwargs.pop('codeVersionId')
if len(kwargs) != 0:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_UNEXPECTED,
kwargs.keys()).message)
config = self._configuration_get()
if codeVersion not in config.codeVersions:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_INVALID,
detail='Invalid codeVersionId').message)
filename = os.path.abspath(os.path.join(self.code_repo, codeVersion))
if not filename.startswith(self.code_repo +
'/') or not os.path.exists(filename):
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_INVALID,
detail='Invalid codeVersionId').message)
return HttpFileDownloadResponse(
config.codeVersions[codeVersion].filename, filename)
def get_node_info(self, kwargs):
if 'serviceNodeId' not in kwargs:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_MISSING,
'serviceNodeId').message)
serviceNodeId = kwargs.pop('serviceNodeId')
if len(kwargs) != 0:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_UNEXPECTED,
kwargs.keys()).message)
config = self._configuration_get()
if serviceNodeId not in config.serviceNodes:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_INVALID,
detail='Invalid "serviceNodeId"').message)
serviceNode = config.serviceNodes[serviceNodeId]
return HttpJsonResponse({
'serviceNode': {
'id': serviceNode.id,
'ip': serviceNode.ip,
'isRunningProxy': serviceNode.isRunningProxy,
'isRunningWeb': serviceNode.isRunningWeb,
'isRunningBackend': serviceNode.isRunningBackend,
}
})
def update_java_configuration(self, kwargs):
if 'codeVersionId' not in kwargs:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_MISSING,
'at least one of "codeVersionId"').message)
codeVersionId = kwargs.pop('codeVersionId')
config = self._configuration_get()
if len(kwargs) != 0:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_UNEXPECTED,
kwargs.keys()).message)
dstate = self._state_get()
if dstate == self.S_INIT or dstate == self.S_STOPPED:
if codeVersionId: config.currentCodeVersion = codeVersionId
self._configuration_set(config)
elif dstate == self.S_RUNNING:
self._state_set(self.S_ADAPTING, msg='Updating configuration')
Thread(target=self.do_update_configuration,
args=[config, codeVersionId]).start()
else:
return HttpErrorResponse(
ManagerException(ManagerException.E_STATE_ERROR).message)
return HttpJsonResponse()
def load_dump(self, kwargs):
self.logger.debug('Uploading mysql dump')
if 'mysqldump_file' not in kwargs:
return HttpErrorResponse(ManagerException(ManagerException.E_ARGS_MISSING, \
'mysqldump_file').message)
mysqldump_file = kwargs.pop('mysqldump_file')
if len(kwargs) != 0:
return HttpErrorResponse(ManagerException(ManagerException.E_ARGS_UNEXPECTED, \
detail='invalid number of arguments ').message)
if not isinstance(mysqldump_file, FileUploadField):
return HttpErrorResponse(ManagerException(ManagerException.E_ARGS_INVALID, \
detail='mysqldump_file should be a file').message)
fd, filename = tempfile.mkstemp(dir='/tmp')
fd = os.fdopen(fd, 'w')
upload = mysqldump_file.file
bytes = upload.read(2048)
while len(bytes) != 0:
fd.write(bytes)
bytes = upload.read(2048)
fd.close()
# Get master
# TODO: modify this when multiple masters
masters = self.config.getMySQLmasters()
try:
for master in masters:
client.load_dump(master.ip, self.config.AGENT_PORT, filename)
except:
self.logger.exception('load_dump: could not upload mysqldump_file ')
self.state = self.S_ERROR
return
return HttpJsonResponse()
def remove_nodes(self, kwargs):
"""Remove kwargs['count'] nodes from this deployment"""
# Removing nodes only if RUNNING
if self.state != self.S_RUNNING:
vals = { 'curstate': self.state, 'action': 'remove_nodes' }
return HttpErrorResponse(self.WRONG_STATE_MSG % vals)
# Ensure 'count' is valid
count_or_err = self.__check_count_in_args(kwargs)
if isinstance(count_or_err, HttpErrorResponse):
return count_or_err
count = count_or_err
if count > len(self.nodes) - 1:
return HttpErrorResponse("ERROR: Cannot remove so many nodes")
self.logger.info(type(kwargs["id"]))
if kwargs["id"] not in self.service.registered_workers.keys():
return HttpErrorResponse("ERROR: This worker does not exist")
id=kwargs["id"]
self.state = self.S_ADAPTING
Thread(target=self._do_remove_nodes, args=[id]).start()
return HttpJsonResponse({ 'state': self.state })
def deleteVolume(self, kwargs):
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to delete Volume')
if not 'volumeName' in kwargs:
return HttpErrorResponse(
'ERROR: Required argument (volumeName) doesn\'t exist')
volumeName = kwargs.pop('volumeName')
args = [
'rmfs.xtreemfs',
'%s:32636/%s' % (self.mrcNodes[0].ip, volumeName)
]
process = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = process.communicate()
process.poll()
if process.returncode != 0:
self.logger.info('Failed to delete volume: %s; %s', stdout, stderr)
return HttpErrorResponse("The volume could not be deleted")
self.logger.info('Deleting Volume: %s; %s', stdout, stderr)
# TODO(maybe): issue xtfs_cleanup on all OSDs to free space (or don't and assume xtfs_cleanup is run by a cron job or something)
return HttpJsonResponse()
def list_nodes(self, kwargs):
if len(kwargs) != 0:
return HttpErrorResponse('ERROR: Arguments unexpected')
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to list_nodes')
return HttpJsonResponse({
'helloworld': [node.id for node in self.nodes],
})
def get_m(self,kwargs):
# self.logger.info(self.configuration.m )
if self.state != self.S_RUNNING:
vals = { 'curstate': self.state, 'action': 'get_cost' }
return HttpErrorResponse(self.WRONG_STATE_MSG % vals)
t = int(kwargs['t'])
if t not in self.configuration.m:
return HttpErrorResponse("manager not configured yet for throughput = "+ str(t))
return HttpJsonResponse({"conf":self.configuration.m[t]})
def list_nodes(self, kwargs):
self.logger.info('called list_nodes')
if len(kwargs) != 0:
return HttpErrorResponse('ERROR: Arguments unexpected')
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to list_nodes')
return HttpJsonResponse({
'scalaris': [ node.id for node in self.nodes ],
})
def upload_code_version(self, kwargs):
if 'code' not in kwargs:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_MISSING,
'code').message)
code = kwargs.pop('code')
if 'description' in kwargs: description = kwargs.pop('description')
else: description = ''
if len(kwargs) != 0:
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_UNEXPECTED,
kwargs.keys()).message)
if not isinstance(code, FileUploadField):
return HttpErrorResponse(
ManagerException(
ManagerException.E_ARGS_INVALID,
detail='codeVersionId should be a file').message)
config = self._configuration_get()
fd, name = tempfile.mkstemp(prefix='code-', dir=self.code_repo)
fd = os.fdopen(fd, 'w')
upload = code.file
codeVersionId = os.path.basename(name)
bytes = upload.read(2048)
while len(bytes) != 0:
fd.write(bytes)
bytes = upload.read(2048)
fd.close()
arch = archive_open(name)
if arch == None:
os.remove(name)
return HttpErrorResponse(
ManagerException(ManagerException.E_ARGS_INVALID,
detail='Invalid archive format').message)
for fname in archive_get_members(arch):
if fname.startswith('/') or fname.startswith('..'):
archive_close(arch)
os.remove(name)
return HttpErrorResponse(
ManagerException(
ManagerException.E_ARGS_INVALID,
detail=
'Absolute file names are not allowed in archive members'
).message)
archive_close(arch)
config.codeVersions[codeVersionId] = CodeVersion(
codeVersionId,
os.path.basename(code.filename),
archive_get_type(name),
description=description)
self._configuration_set(config)
return HttpJsonResponse(
{'codeVersionId': os.path.basename(codeVersionId)})
def list_nodes(self, kwargs):
if len(kwargs) != 0:
return HttpErrorResponse('ERROR: Arguments unexpected')
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to list_nodes')
return HttpJsonResponse({
'dir': [node.id for node in self.dirNodes],
'mrc': [node.id for node in self.mrcNodes],
'osd': [node.id for node in self.osdNodes]
})
def set_osd_size(self, kwargs):
if not 'size' in kwargs:
return HttpErrorResponse(
"ERROR: Required argument (size) doesn't exist")
try:
self.osd_volume_size = int(kwargs['size'])
self.logger.debug('set_osd_size: %s' % self.osd_volume_size)
return self.get_service_info({})
except ValueError:
return HttpErrorResponse(
"ERROR: Required argument (size) should be an integer")
def remove_nodes(self, kwargs):
self.logger.info('called remove_nodes')
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to remove_nodes')
if not 'scalaris' in kwargs:
return HttpErrorResponse('ERROR: Required argument doesn\'t exist')
if not isinstance(kwargs['scalaris'], int):
return HttpErrorResponse('ERROR: Expected an integer value for "count"')
count = int(kwargs.pop('scalaris'))
self.state = self.S_ADAPTING
Thread(target=self._do_remove_nodes, args=[count]).start()
return HttpJsonResponse()
def updateHttpProxy(self, kwargs):
try:
kwargs = self._httpproxy_get_params(kwargs)
with self.httpproxy_lock:
return self._update(kwargs, self.httpproxy_file,
self.HttpProxy)
except AgentException as e:
self.logger.exception(e)
return HttpErrorResponse(e.message)
except Exception as e:
self.logger.exception(e)
return HttpErrorResponse(str(e))
def sqldump(self, kwargs):
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to call sqldump')
master_ip = self.config.getMySQLmasters()[0].ip
cmd = 'mysqldump -u mysqldb -h %s --password=%s -A' % (master_ip,
self.root_pass)
out, err = run_cmd(cmd)
if err:
return HttpErrorResponse(err)
return HttpJsonResponse(out)
def add_nodes(self, kwargs):
self.controller.update_context(dict(STRING='helloworld'))
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to add_nodes')
if not 'count' in kwargs:
return HttpErrorResponse('ERROR: Required argument doesn\'t exist')
if not isinstance(kwargs['count'], int):
return HttpErrorResponse(
'ERROR: Expected an integer value for "count"')
count = int(kwargs.pop('count'))
self.state = self.S_ADAPTING
Thread(target=self._do_add_nodes, args=[count]).start()
return HttpJsonResponse()
def unsubscribe(self, params):
if not 'origin' in params:
return HttpErrorResponse('Missing parameter: "origin"')
origin = params['origin']
try:
filename = os.path.join(self.apps_dir, origin)
if not os.access(filename, os.R_OK):
return HttpErrorResponse('"%s" is not subscribed' % (origin))
os.remove(filename)
return HttpJsonResponse({'unsubscribe': True})
except Exception as e:
return HttpErrorResponse('Error unsubscribing "%s": %s' %
(origin, str(e)))
def remove_nodes(self, kwargs):
if self.state != self.S_RUNNING:
self.logger.debug('Wrong state to remove nodes')
return HttpErrorResponse('ERROR: Wrong state to remove_nodes')
if not 'slaves' in kwargs:
return HttpErrorResponse('ERROR: Required argument doesn\'t exist')
if not isinstance(kwargs['slaves'], int):
return HttpErrorResponse('ERROR: Expected an integer value for "count"')
count = int(kwargs.pop('slaves'))
if count > len(self.config.getMySQLslaves()):
return HttpErrorResponse('ERROR: Cannot remove so many nodes')
self.state = self.S_ADAPTING
Thread(target=self._do_remove_nodes, args=[count]).start()
return HttpJsonResponse()
def __check_count_in_args(self, kwargs):
"""Return 'count' if all is good. HttpErrorResponse otherwise."""
# The frontend sends count under 'node'.
if 'node' in kwargs:
kwargs['count'] = kwargs['node']
if not 'count' in kwargs:
return HttpErrorResponse(self.REQUIRED_ARG_MSG % {'arg': 'count'})
if not isinstance(kwargs['count'], int):
return HttpErrorResponse(
"ERROR: Expected an integer value for 'count'")
return int(kwargs['count'])
def add_nodes(self, kwargs):
if self.state != self.S_RUNNING:
return HttpErrorResponse('ERROR: Wrong state to add_nodes')
if not 'scalaris' in kwargs:
return HttpErrorResponse('ERROR: Required argument doesn\'t exist')
if not isinstance(kwargs['scalaris'], int):
return HttpErrorResponse('ERROR: Expected an integer value for "count"')
count = int(kwargs.pop('scalaris'))
# create at least one node
if count < 1:
return HttpErrorResponse('ERROR: Expected a positive integer value for "count"')
self.state = self.S_ADAPTING
Thread(target=self._do_add_nodes, args=[count, kwargs['cloud']]).start()
return HttpJsonResponse()
