def get_node_info(self, kwargs): """ HTTP GET method. Gets info of a specific node. :param param: serviceNodeId is a VMID of an existing service node. :type param: str :returns: HttpJsonResponse - JSON response with details about the node. :raises: ManagerException """ if 'serviceNodeId' not in kwargs: return HttpErrorResponse(ManagerException(E_ARGS_MISSING, 'serviceNodeId').message) serviceNodeId = kwargs.pop('serviceNodeId') if len(kwargs) != 0: return HttpErrorResponse(ManagerException(E_ARGS_UNEXPECTED, kwargs.keys()).message) if serviceNodeId not in self.config.serviceNodes: return HttpErrorResponse(ManagerException(E_ARGS_INVALID , \ "serviceNodeId" ,\ detail='Invalid "serviceNodeId"').message) serviceNode = self.config.getMySQLNode(serviceNodeId) return HttpJsonResponse({ 'serviceNode': { 'id': serviceNode.id, 'ip': serviceNode.ip, 'isMaster': serviceNode.isMaster, 'isSlave': serviceNode.isSlave } })
def _update(self, post_params, class_file, pClass): try: if type(post_params) != dict: raise TypeError() fd = open(class_file, 'r') p = pickle.load(fd) fd.close() p.configure(**post_params) p.restart() except (ValueError, TypeError) as e: self.logger.exception(e) ex = AgentException(AgentException.E_ARGS_INVALID) return HttpErrorResponse(ex.message) except Exception as e: self.logger.exception(e) ex = AgentException(AgentException.E_UNKNOWN, detail=e) return HttpErrorResponse(ex.message) else: try: fd = open(class_file, 'w') pickle.dump(p, fd) fd.close() except Exception as e: self.logger.exception(ex.message) ex = AgentException(AgentException.E_CONFIG_COMMIT_FAILED, detail=e) return HttpErrorResponse(ex.message) else: return HttpJsonResponse()
def get_node_info(self, kwargs): """Return information about the node identified by the given kwargs['serviceNodeId']""" # serviceNodeId is a required parameter if 'serviceNodeId' not in kwargs: vals = {'arg': 'serviceNodeId'} return HttpErrorResponse(self.REQUIRED_ARG_MSG % vals) serviceNodeId = kwargs.pop('serviceNodeId') serviceNode = None for node in self.nodes: if serviceNodeId == node.id: serviceNode = node break if serviceNode is None: return HttpErrorResponse( 'ERROR: Cannot find node with serviceNode=%s' % serviceNodeId) return HttpJsonResponse({ 'serviceNode': { 'id': serviceNode.id, 'ip': serviceNode.ip, 'is_hub': self.__is_hub(serviceNode) } })
def _stop(self, kwargs, class_file, pClass): self.logger.debug("_stop(kwargs=%s, class_file=%s, pClass=%s)" % (kwargs, class_file, pClass)) if not exists(class_file): self.logger.error("class_file '%s' does not exist" % class_file) return HttpErrorResponse( AgentException(AgentException.E_CONFIG_NOT_EXIST).message) try: try: fd = open(class_file, 'r') p = pickle.load(fd) self.logger.debug('dump file %s loaded' % class_file) fd.close() except Exception as e: ex = AgentException(AgentException.E_CONFIG_READ_FAILED, detail=e) self.logger.exception(ex.message) return HttpErrorResponse(ex.message) if 'drain' in kwargs: p.stop(kwargs['drain']) else: p.stop() self.logger.debug("Removing class_file '%s'" % class_file) remove(class_file) return HttpJsonResponse() except Exception as e: ex = AgentException(AgentException.E_UNKNOWN, detail=e) self.logger.exception(e) return HttpErrorResponse(ex.message)
def set_striping_policy(self, kwargs): if self.state != self.S_RUNNING: return HttpErrorResponse( 'ERROR: Wrong state to set Striping policy.') if not 'volumeName' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (volumeName) doesn\'t exist') if not 'policy' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (policy) doesn\'t exist') if not 'width' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (factor) doesn\'t exist') if not 'stripe-size' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (stripe-size) doesn\'t exist') volumeName = kwargs.pop('volumeName') policy = kwargs.pop('policy') width = kwargs.pop('width') stripe_size = kwargs.pop('stripe-size') # xtfsutil <path> --set-dsp --striping-policy <policy> --striping-policy-width <width> --striping-policy-stripe-size <stripe-size> args = [ '--set-dsp', '--striping-policy', policy, '--striping-policy-width', width, '--striping-policy-stripe-size', stripe_size ] return self.set_policy(volumeName, 'Striping', args)
def set_password(self, kwargs): self.logger.debug('Setting password') if self.state != self.S_RUNNING: self.logger.debug('Service not runnning') return HttpErrorResponse('ERROR: Service not running') if not 'user' in kwargs: return HttpErrorResponse('ERROR: Required argument \'user\' doesn\'t exist') if not 'password' in kwargs: return HttpErrorResponse('ERROR: Required argument \'password\' doesn\'t exist') # Get the master masters = self.config.getMySQLmasters() #TODO: modify this when multiple masters try: for master in masters: client.set_password(master.ip, self.config.AGENT_PORT, kwargs['user'], kwargs['password']) except: e = sys.exc_info()[1] self.logger.exception('set_password: Could not set password: %s' % e) self.state = self.S_ERROR return HttpErrorResponse('Failed to set password') else: self.root_pass = kwargs['password'] return HttpJsonResponse()
def set_replication_policy(self, kwargs): if self.state != self.S_RUNNING: return HttpErrorResponse( 'ERROR: Wrong state to set Replication policy.') if not 'volumeName' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (volumeName) doesn\'t exist') if not 'policy' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (policy) doesn\'t exist') if not 'factor' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (factor) doesn\'t exist') volumeName = kwargs.pop('volumeName') policy = kwargs.pop('policy') factor = kwargs.pop('factor') # xtfsutil <path> --set-drp --replication-policy <policy> --replication-factor <factor> args = [ '--set-drp', '--replication-policy', policy, '--replication-factor', factor ] return self.set_policy(volumeName, 'Replication', args)
def _create(self, post_params, class_file, pClass): if exists(class_file): return HttpErrorResponse( AgentException(AgentException.E_CONFIG_EXISTS).message) try: if type(post_params) != dict: raise TypeError() self.logger.debug('Creating class') p = pClass(**post_params) self.logger.debug('Created class') except (ValueError, TypeError) as e: ex = AgentException(AgentException.E_ARGS_INVALID, detail=str(e)) self.logger.exception(e) return HttpErrorResponse(ex.message) except Exception as e: ex = AgentException(AgentException.E_UNKNOWN, detail=e) self.logger.exception(e) return HttpErrorResponse(ex.message) else: try: self.logger.debug('Openning file %s' % class_file) fd = open(class_file, 'w') pickle.dump(p, fd) fd.close() except Exception as e: ex = AgentException(AgentException.E_CONFIG_COMMIT_FAILED, detail=e) self.logger.exception(ex.message) return HttpErrorResponse(ex.message) else: self.logger.debug('Created class file') return HttpJsonResponse()
def createVolume(self, kwargs): if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to create Volume') if not 'volumeName' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (volumeName) doesn\'t exist') volumeName = kwargs.pop('volumeName') # Get the value of 'owner', if specified. 'xtreemfs' otherwise owner = kwargs.pop('owner', 'xtreemfs') args = [ 'mkfs.xtreemfs', '%s:32636/%s' % (self.mrcNodes[0].ip, volumeName), "-u", owner, "-g", owner, "-m", "777" ] process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = process.communicate() process.poll() if process.returncode != 0: self.logger.info('Failed to create volume: %s; %s', stdout, stderr) return HttpErrorResponse("The volume could not be created") self.logger.info('Creating Volume: %s; %s', stdout, stderr) return HttpJsonResponse()
def startup(self, kwargs): self.logger.info("HTC Manager starting up") self.logger.info(str(kwargs)) if self.state != self.S_INIT and self.state != self.S_STOPPED: vals = { 'curstate': self.state, 'action': 'startup' } return HttpErrorResponse(self.WRONG_STATE_MSG % vals) if 'cloud' in kwargs: try: self._init_cloud(kwargs['cloud']) except Exception: return HttpErrorResponse( "A cloud named '%s' could not be found" % kwargs['cloud']) #self.logger.info('Get service TaskFarm') try: self.service = TaskFarm(kwargs['mode'], kwargs['type']) except (UnknownHtcTypeError, UnimplementedHtcCombinationError) as e: return HttpErrorResponse({ 'error': e.__str__() }) #self.logger.info('Got service TaskFarm, delete some kwargs entries') self.state = self.S_PROLOGUE del kwargs['type'] del kwargs['mode'] #del kwargs['m_type'] #self.logger.info('Show leftover kwargs entries') self.logger.info(str(kwargs)) #self.logger.info('Starting Thread for startup') Thread(target=self._do_startup, kwargs=kwargs).start() self.logger.info(str(self.service)) return HttpJsonResponse({ 'state': self.state })
def download_code_version(self, kwargs): if 'codeVersionId' not in kwargs: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_MISSING, 'codeVersionId').message) if isinstance(kwargs['codeVersionId'], dict): return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='codeVersionId should be a string').message) codeVersion = kwargs.pop('codeVersionId') if len(kwargs) != 0: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_UNEXPECTED, kwargs.keys()).message) config = self._configuration_get() if codeVersion not in config.codeVersions: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_INVALID, detail='Invalid codeVersionId').message) filename = os.path.abspath(os.path.join(self.code_repo, codeVersion)) if not filename.startswith(self.code_repo + '/') or not os.path.exists(filename): return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_INVALID, detail='Invalid codeVersionId').message) return HttpFileDownloadResponse( config.codeVersions[codeVersion].filename, filename)
def get_node_info(self, kwargs): if 'serviceNodeId' not in kwargs: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_MISSING, 'serviceNodeId').message) serviceNodeId = kwargs.pop('serviceNodeId') if len(kwargs) != 0: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_UNEXPECTED, kwargs.keys()).message) config = self._configuration_get() if serviceNodeId not in config.serviceNodes: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_INVALID, detail='Invalid "serviceNodeId"').message) serviceNode = config.serviceNodes[serviceNodeId] return HttpJsonResponse({ 'serviceNode': { 'id': serviceNode.id, 'ip': serviceNode.ip, 'isRunningProxy': serviceNode.isRunningProxy, 'isRunningWeb': serviceNode.isRunningWeb, 'isRunningBackend': serviceNode.isRunningBackend, } })
def update_java_configuration(self, kwargs): if 'codeVersionId' not in kwargs: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_MISSING, 'at least one of "codeVersionId"').message) codeVersionId = kwargs.pop('codeVersionId') config = self._configuration_get() if len(kwargs) != 0: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_UNEXPECTED, kwargs.keys()).message) dstate = self._state_get() if dstate == self.S_INIT or dstate == self.S_STOPPED: if codeVersionId: config.currentCodeVersion = codeVersionId self._configuration_set(config) elif dstate == self.S_RUNNING: self._state_set(self.S_ADAPTING, msg='Updating configuration') Thread(target=self.do_update_configuration, args=[config, codeVersionId]).start() else: return HttpErrorResponse( ManagerException(ManagerException.E_STATE_ERROR).message) return HttpJsonResponse()
def load_dump(self, kwargs): self.logger.debug('Uploading mysql dump') if 'mysqldump_file' not in kwargs: return HttpErrorResponse(ManagerException(ManagerException.E_ARGS_MISSING, \ 'mysqldump_file').message) mysqldump_file = kwargs.pop('mysqldump_file') if len(kwargs) != 0: return HttpErrorResponse(ManagerException(ManagerException.E_ARGS_UNEXPECTED, \ detail='invalid number of arguments ').message) if not isinstance(mysqldump_file, FileUploadField): return HttpErrorResponse(ManagerException(ManagerException.E_ARGS_INVALID, \ detail='mysqldump_file should be a file').message) fd, filename = tempfile.mkstemp(dir='/tmp') fd = os.fdopen(fd, 'w') upload = mysqldump_file.file bytes = upload.read(2048) while len(bytes) != 0: fd.write(bytes) bytes = upload.read(2048) fd.close() # Get master # TODO: modify this when multiple masters masters = self.config.getMySQLmasters() try: for master in masters: client.load_dump(master.ip, self.config.AGENT_PORT, filename) except: self.logger.exception('load_dump: could not upload mysqldump_file ') self.state = self.S_ERROR return return HttpJsonResponse()
def remove_nodes(self, kwargs): """Remove kwargs['count'] nodes from this deployment""" # Removing nodes only if RUNNING if self.state != self.S_RUNNING: vals = { 'curstate': self.state, 'action': 'remove_nodes' } return HttpErrorResponse(self.WRONG_STATE_MSG % vals) # Ensure 'count' is valid count_or_err = self.__check_count_in_args(kwargs) if isinstance(count_or_err, HttpErrorResponse): return count_or_err count = count_or_err if count > len(self.nodes) - 1: return HttpErrorResponse("ERROR: Cannot remove so many nodes") self.logger.info(type(kwargs["id"])) if kwargs["id"] not in self.service.registered_workers.keys(): return HttpErrorResponse("ERROR: This worker does not exist") id=kwargs["id"] self.state = self.S_ADAPTING Thread(target=self._do_remove_nodes, args=[id]).start() return HttpJsonResponse({ 'state': self.state })
def deleteVolume(self, kwargs): if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to delete Volume') if not 'volumeName' in kwargs: return HttpErrorResponse( 'ERROR: Required argument (volumeName) doesn\'t exist') volumeName = kwargs.pop('volumeName') args = [ 'rmfs.xtreemfs', '%s:32636/%s' % (self.mrcNodes[0].ip, volumeName) ] process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = process.communicate() process.poll() if process.returncode != 0: self.logger.info('Failed to delete volume: %s; %s', stdout, stderr) return HttpErrorResponse("The volume could not be deleted") self.logger.info('Deleting Volume: %s; %s', stdout, stderr) # TODO(maybe): issue xtfs_cleanup on all OSDs to free space (or don't and assume xtfs_cleanup is run by a cron job or something) return HttpJsonResponse()
def list_nodes(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpected') if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to list_nodes') return HttpJsonResponse({ 'helloworld': [node.id for node in self.nodes], })
def get_m(self,kwargs): # self.logger.info(self.configuration.m ) if self.state != self.S_RUNNING: vals = { 'curstate': self.state, 'action': 'get_cost' } return HttpErrorResponse(self.WRONG_STATE_MSG % vals) t = int(kwargs['t']) if t not in self.configuration.m: return HttpErrorResponse("manager not configured yet for throughput = "+ str(t)) return HttpJsonResponse({"conf":self.configuration.m[t]})
def list_nodes(self, kwargs): self.logger.info('called list_nodes') if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpected') if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to list_nodes') return HttpJsonResponse({ 'scalaris': [ node.id for node in self.nodes ], })
def upload_code_version(self, kwargs): if 'code' not in kwargs: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_MISSING, 'code').message) code = kwargs.pop('code') if 'description' in kwargs: description = kwargs.pop('description') else: description = '' if len(kwargs) != 0: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_UNEXPECTED, kwargs.keys()).message) if not isinstance(code, FileUploadField): return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='codeVersionId should be a file').message) config = self._configuration_get() fd, name = tempfile.mkstemp(prefix='code-', dir=self.code_repo) fd = os.fdopen(fd, 'w') upload = code.file codeVersionId = os.path.basename(name) bytes = upload.read(2048) while len(bytes) != 0: fd.write(bytes) bytes = upload.read(2048) fd.close() arch = archive_open(name) if arch == None: os.remove(name) return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_INVALID, detail='Invalid archive format').message) for fname in archive_get_members(arch): if fname.startswith('/') or fname.startswith('..'): archive_close(arch) os.remove(name) return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail= 'Absolute file names are not allowed in archive members' ).message) archive_close(arch) config.codeVersions[codeVersionId] = CodeVersion( codeVersionId, os.path.basename(code.filename), archive_get_type(name), description=description) self._configuration_set(config) return HttpJsonResponse( {'codeVersionId': os.path.basename(codeVersionId)})
def list_nodes(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpected') if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to list_nodes') return HttpJsonResponse({ 'dir': [node.id for node in self.dirNodes], 'mrc': [node.id for node in self.mrcNodes], 'osd': [node.id for node in self.osdNodes] })
def set_osd_size(self, kwargs): if not 'size' in kwargs: return HttpErrorResponse( "ERROR: Required argument (size) doesn't exist") try: self.osd_volume_size = int(kwargs['size']) self.logger.debug('set_osd_size: %s' % self.osd_volume_size) return self.get_service_info({}) except ValueError: return HttpErrorResponse( "ERROR: Required argument (size) should be an integer")
def remove_nodes(self, kwargs): self.logger.info('called remove_nodes') if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to remove_nodes') if not 'scalaris' in kwargs: return HttpErrorResponse('ERROR: Required argument doesn\'t exist') if not isinstance(kwargs['scalaris'], int): return HttpErrorResponse('ERROR: Expected an integer value for "count"') count = int(kwargs.pop('scalaris')) self.state = self.S_ADAPTING Thread(target=self._do_remove_nodes, args=[count]).start() return HttpJsonResponse()
def updateHttpProxy(self, kwargs): try: kwargs = self._httpproxy_get_params(kwargs) with self.httpproxy_lock: return self._update(kwargs, self.httpproxy_file, self.HttpProxy) except AgentException as e: self.logger.exception(e) return HttpErrorResponse(e.message) except Exception as e: self.logger.exception(e) return HttpErrorResponse(str(e))
def sqldump(self, kwargs): if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to call sqldump') master_ip = self.config.getMySQLmasters()[0].ip cmd = 'mysqldump -u mysqldb -h %s --password=%s -A' % (master_ip, self.root_pass) out, err = run_cmd(cmd) if err: return HttpErrorResponse(err) return HttpJsonResponse(out)
def add_nodes(self, kwargs): self.controller.update_context(dict(STRING='helloworld')) if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to add_nodes') if not 'count' in kwargs: return HttpErrorResponse('ERROR: Required argument doesn\'t exist') if not isinstance(kwargs['count'], int): return HttpErrorResponse( 'ERROR: Expected an integer value for "count"') count = int(kwargs.pop('count')) self.state = self.S_ADAPTING Thread(target=self._do_add_nodes, args=[count]).start() return HttpJsonResponse()
def unsubscribe(self, params): if not 'origin' in params: return HttpErrorResponse('Missing parameter: "origin"') origin = params['origin'] try: filename = os.path.join(self.apps_dir, origin) if not os.access(filename, os.R_OK): return HttpErrorResponse('"%s" is not subscribed' % (origin)) os.remove(filename) return HttpJsonResponse({'unsubscribe': True}) except Exception as e: return HttpErrorResponse('Error unsubscribing "%s": %s' % (origin, str(e)))
def remove_nodes(self, kwargs): if self.state != self.S_RUNNING: self.logger.debug('Wrong state to remove nodes') return HttpErrorResponse('ERROR: Wrong state to remove_nodes') if not 'slaves' in kwargs: return HttpErrorResponse('ERROR: Required argument doesn\'t exist') if not isinstance(kwargs['slaves'], int): return HttpErrorResponse('ERROR: Expected an integer value for "count"') count = int(kwargs.pop('slaves')) if count > len(self.config.getMySQLslaves()): return HttpErrorResponse('ERROR: Cannot remove so many nodes') self.state = self.S_ADAPTING Thread(target=self._do_remove_nodes, args=[count]).start() return HttpJsonResponse()
def __check_count_in_args(self, kwargs): """Return 'count' if all is good. HttpErrorResponse otherwise.""" # The frontend sends count under 'node'. if 'node' in kwargs: kwargs['count'] = kwargs['node'] if not 'count' in kwargs: return HttpErrorResponse(self.REQUIRED_ARG_MSG % {'arg': 'count'}) if not isinstance(kwargs['count'], int): return HttpErrorResponse( "ERROR: Expected an integer value for 'count'") return int(kwargs['count'])
def add_nodes(self, kwargs): if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to add_nodes') if not 'scalaris' in kwargs: return HttpErrorResponse('ERROR: Required argument doesn\'t exist') if not isinstance(kwargs['scalaris'], int): return HttpErrorResponse('ERROR: Expected an integer value for "count"') count = int(kwargs.pop('scalaris')) # create at least one node if count < 1: return HttpErrorResponse('ERROR: Expected a positive integer value for "count"') self.state = self.S_ADAPTING Thread(target=self._do_add_nodes, args=[count, kwargs['cloud']]).start() return HttpJsonResponse()