def _do_add_nodes(self, nr_dir, nr_mrc, nr_osd, cloud, resuming=False): startCloud = self._init_cloud(cloud) totalNodes = nr_dir + nr_mrc + nr_osd # try to create totalNodes new nodes try: node_instances = self.controller.create_nodes( totalNodes, client.check_agent_process, 5555, startCloud) except: self.logger.exception( '_do_add_nodes: Failed to request a new node') self.state = self.S_STOPPED return self.nodes += node_instances dirNodesAdded = node_instances[:nr_dir] self.dirNodes += dirNodesAdded mrcNodesAdded = node_instances[nr_dir:nr_mrc + nr_dir] self.mrcNodes += mrcNodesAdded osdNodesAdded = node_instances[nr_mrc + nr_dir:] self.osdNodes += osdNodesAdded # TODO: maybe re-enable when OSD-removal moves data to another node before shutting down the service. #KilledOsdNodes = [] # The first node will contain the OSD service so it will be removed # from there #if nr_osd > 0 and self.osdCount == 0: # KilledOsdNodes.append(self.dirNodes[0]) #self.KillOsd(KilledOsdNodes) # Startup DIR agents for node in dirNodesAdded: client.startup(node.ip, 5555) data = client.createDIR(node.ip, 5555) self.logger.info('Received %s from %s', data, node.id) self.dirCount += 1 # Startup MRC agents for node in mrcNodesAdded: client.startup(node.ip, 5555) data = client.createMRC(node.ip, 5555, self.dirNodes[0].ip) self.logger.info('Received %s from %s', data, node.id) self.mrcCount += 1 # Startup OSD agents (if not resuming) if not resuming: self._start_osd(osdNodesAdded, startCloud) self.osdCount += len(osdNodesAdded) #for node in osdNodesAdded: # client.startup(node.ip, 5555) # data = client.createOSD(node.ip, 5555, self.dirNodes[0].ip) # self.logger.info('Received %s from %s', data, node.id) # self.osdCount += 1 self.state = self.S_RUNNING return HttpJsonResponse()
def shutdown(self, kwargs): self.state = self.S_EPILOGUE Thread(target=self._do_shutdown, args=[]).start() return HttpJsonResponse()
def _do_remove_nodes(self, count): nodes = self.config.getMySQLslaves()[:count] self.controller.delete_nodes(nodes) self.config.remove_nodes(nodes) self.state = self.S_RUNNING return HttpJsonResponse()
def shutdown(self, kwargs): self.state = self.S_EPILOGUE # start _do_shutdown(stop_services=True) in a thread Thread(target=self._do_shutdown, args=[True]).start() return HttpJsonResponse()
def set_policy(self, volumeName, policyName, args): mountPoint = '/tmp/' + volumeName # mkdir -p <mountpoint> process = subprocess.Popen(['mkdir', '-p', mountPoint]) (stdout, stderr) = process.communicate() process.poll() if process.returncode != 0: self.logger.info('Failed to set %s policy: %s; %s', policyName, stdout, stderr) return HttpErrorResponse("Failed to set %s policy: %s; %s" % (policyName, stdout, stderr)) # mount.xtreemfs <dir_ip>:32638/<volumename> <mountpoint> process = subprocess.Popen([ 'mount.xtreemfs', '%s:32638/%s' % (self.dirNodes[0].ip, volumeName), mountPoint ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = process.communicate() process.poll() if process.returncode != 0: self.logger.info('Failed to set %s policy: %s; %s', policyName, stdout, stderr) return HttpErrorResponse("Failed to set %s policy: %s; %s" % (policyName, stdout, stderr)) # # with python 2.7 # try: # # mkdir -p <mountpoint> # subprocess.check_output(['mkdir', '-p', mountPoint]) # # mount.xtreemfs <dir_ip>:32638/<volumename> <mountpoint> # subprocess.check_output(['mount.xtreemfs', # '%s:32638/%s' % (self.dirNodes[0].ip, volumeName), # mountPoint], # stdout=subprocess.STDOUT) # except subprocess.CalledProcessError as e: # return HttpErrorResponse('ERROR: could not mount volume: ' + e.output) # xtfsutil <mountpoint> args process = subprocess.Popen(['xtfsutil', mountPoint] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout_xtfsutil, stderr_xtfsutil) = (stdout, stderr) = process.communicate() process.poll() if process.returncode != 0: self.logger.info('Failed to set %s policy: %s; %s', policyName, stdout, stderr) return HttpErrorResponse("Failed to set %s policy: %s; %s" % (policyName, stdout, stderr)) # umount <mountpoint> process = subprocess.Popen(['umount', mountPoint], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = process.communicate() process.poll() if process.returncode != 0: self.logger.info('Failed to set %s policy: %s; %s', policyName, stdout, stderr) return HttpErrorResponse("Failed to set %s policy: %s; %s" % (policyName, stdout, stderr)) # rmdir <mountpoint> process = subprocess.Popen(['rmdir', mountPoint]) (stdout, stderr) = process.communicate() process.poll() if process.returncode != 0: self.logger.info('Failed to set %s policy: %s; %s', policyName, stdout, stderr) return HttpErrorResponse("Failed to set %s policy: %s; %s" % (policyName, stdout, stderr)) # # with python 2.7 # try: # # umount <mountpoint> # subprocess.check_output(['umount', mountPoint]) # # fusermount -u <mountpoint> # #subprocess.check_output(['fusermount', '-u', mountPoint]) # # rmdir <mountpoint> # subprocess.check_output(['rmdir', mountPoint]) # except subprocess.CalledProcessError as e: # return HttpErrorResponse('ERROR: could not unmount volume: ' + e.output) self.logger.info('Setting %s policy: %s; %s', policyName, stdout_xtfsutil, stderr_xtfsutil) return HttpJsonResponse({'stdout': stdout_xtfsutil})
def get_service_history(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_UNEXPECTED, kwargs.keys()).message) return HttpJsonResponse({'state_log': self.state_log})
def startup(self, kwargs): self.state = 'RUNNING' self.logger.info('Agent started up') return HttpJsonResponse()
def list_replication_policies(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpetced') return HttpJsonResponse({'policies': "ronly, WaR1, WqRq"})
def _do_remove_nodes(self, count): for i in range(0, count): self.controller.delete_nodes([self.nodes.pop(0)]) self.state = self.S_RUNNING return HttpJsonResponse()
def startup(self, kwargs): self.logger.info('Manager started up') self.state = self.S_RUNNING return HttpJsonResponse()
def check_agent_process(self, kwargs): """Check if agent process started - just return an empty response""" if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpected') return HttpJsonResponse()
def get_helloworld(self, kwargs): if self.state != 'RUNNING': return HttpErrorResponse('ERROR: Wrong state to get_helloworld') return HttpJsonResponse({'result': self.gen_string})
def get_password(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpected') return HttpJsonResponse(self.root_pass)
def get_service_info(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpected') return HttpJsonResponse({'state': self.state, 'type': 'xtreemfs'})
def get_service_info(self, kwargs): self.logger.info('called get_service_info: %s', self.state) if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpected') return HttpJsonResponse({'state': self.state, 'type': 'mapreduce'})
def list_striping_policies(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpetced') return HttpJsonResponse({'policies': "RAID0"})
def _do_shutdown(self): self.controller.delete_nodes(self.nodes) self.nodes = [] self.state = self.S_STOPPED return HttpJsonResponse()
def list_replica_sel_policies(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse('ERROR: Arguments unexpetced') return HttpJsonResponse({'policies': "DEFAULT, FQDN, DCMAP, VIVALDI"})
def get_service_info(self, kwargs): if len(kwargs) != 0: return HttpErrorResponse( ManagerException(ManagerException.E_ARGS_UNEXPECTED, kwargs.keys()).message) return HttpJsonResponse({'state': self._state_get(), 'type': 'PHP'})
def get_service_info(self, kwargs): """Return the service state and type""" return HttpJsonResponse({'state': self.state, 'type': 'taskfarm'})
self.osdNodes.append(node) if volumeid: self.osd_uuid_volume_map[osd_uuid] = volumeid try: self.get_volume(volumeid) except Exception: # This volume is not in the list of known ones. volumeCloud = self._init_cloud(data.get('cloud')) class volume: id = volumeid cloud = volumeCloud self.volumes.append(volume) # Regardless of node type, restore metadata try: self.logger.info('set_service_snapshot: restoring %s' % node.ip) data = client.set_snapshot(node.ip, 5555, data['archive']) except client.AgentException, err: self.logger.exception(err) raise err self.logger.info("set_service_snapshot: starting all agent services") self._start_all() self.logger.info("set_service_snapshot: all agent services started") return HttpJsonResponse()
def add_nodes(self, kwargs): config = self._configuration_get() dstate = self._state_get() if dstate != self.S_RUNNING: return HttpErrorResponse( ManagerException(ManagerException.E_STATE_ERROR).message) backend = 0 web = 0 proxy = 0 vm_backend_type = None vm_web_type = None if 'backend' in kwargs: if not isinstance(kwargs['backend'], int): return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='Expected an integer value for "backend"'). message) backend = int(kwargs.pop('backend')) if backend < 0: return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='Expected a positive integer value for "backend"' ).message) if 'web' in kwargs: if not isinstance(kwargs['web'], int): return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='Expected an integer value for "web"').message) web = int(kwargs.pop('web')) if web < 0: return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='Expected a positive integer value for "web"'). message) if 'proxy' in kwargs: if not isinstance(kwargs['proxy'], int): return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='Expected an integer value for "proxy"').message ) proxy = int(kwargs.pop('proxy')) if proxy < 0: return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='Expected a positive integer value for "proxy"' ).message) if (backend + web + proxy) < 1: return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_MISSING, ['backend', 'web', 'proxy'], detail='Need a positive value for at least one').message) ####### ADDED SCALING V2: ###################### if 'vm_backend_instance' in kwargs: self.logger.info('VM BACKEND INSTACE: %s' % str(kwargs['vm_backend_instance'])) if not isinstance(str(kwargs['vm_backend_instance']), basestring): return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail= 'Expected a string value for "vm_backend_instance"'). message) vm_backend_type = kwargs.pop('vm_backend_instance') if len(vm_backend_type) <= 0: return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail= 'Expected a string value for "vm_backend_instance"'). message) if 'vm_web_instance' in kwargs: self.logger.info('VM WEB INSTACE: %s' % str(kwargs['vm_web_instance'])) if not isinstance(str(kwargs['vm_web_instance']), basestring): return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='Expected a string value for "vm_web_instance"' ).message) vm_web_type = kwargs.pop('vm_web_instance') if len(vm_web_type) <= 0: return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail='Expected a string value for "vm_web_instance"' ).message) ############################# if (proxy + config.proxy_count) > 1 and ( (web + config.web_count) == 0 or (backend + config.backend_count) == 0): return HttpErrorResponse( ManagerException( ManagerException.E_ARGS_INVALID, detail= 'Cannot add more proxy servers without at least one "web" and one "backend"' ).message) self._state_set( self.S_ADAPTING, msg= 'Going to add proxy=%d, web=%d, backend=%d, vm_backend_type=%s, vm_web_type=%s, cloud=%s' % (proxy, web, backend, str(vm_backend_type), str(vm_web_type), str(kwargs['cloud']))) Thread(target=self.do_add_nodes, args=[ config, proxy, web, backend, kwargs['cloud'], vm_backend_type, vm_web_type ]).start() return HttpJsonResponse()
def getLog(self, kwargs): """Return logfile""" try: return HttpJsonResponse({'log': open(self.logfile).read()}) except: return HttpErrorResponse('Failed to read log')