def stop(self, drain = True): if self.state == S_RUNNING: self.state = S_STOPPING devnull_fd = open(devnull,'w') if drain: # remove (drain) OSD to avoid data loss logger.info('OSD Service remove commandline: ' + str(self.remove_args)) logger.info('OSD Service stop commandline: ' + str(self.stop_args)) proc = Popen(self.remove_args, stdout = devnull_fd, stderr = devnull_fd, close_fds = True) if proc.wait() != 0: logger.critical('Failed to remove OSD service:(code=%d)' %proc.returncode) logger.info('OSD Service removed.') # stop OSD proc = Popen(self.stop_args, stdout = devnull_fd, stderr = devnull_fd, close_fds = True) if proc.wait() != 0: logger.critical('Failed to stop OSD service:(code=%d)' %proc.returncode) logger.info('OSD Service stopped.') # kill all processes still using the storage block device fuser_cmd = ' '.join(self.fuser_args) logger.debug("Running command '%s'" % fuser_cmd) run_cmd(fuser_cmd) # unmount storage block device proc = Popen(self.umount_args, stdout = devnull_fd, stderr = devnull_fd, close_fds = True) if proc.wait() != 0: logger.critical('Failed to unmount storage device:(code=%d)' %proc.returncode) logger.info('Storage block device unmounted.') self.state = S_STOPPED else: logger.warning('Request to stop OSD service while it is not running')
def configure(self): """Here we also need to configure gmetad and the ganglia frontend""" BaseGanglia.configure(self) # Write gmetad.conf src = open(os.path.join(self.cps_home, 'config', 'ganglia', 'ganglia-gmetad.tmpl')).read() tmpl = Template(src, { 'clusterName': self.cluster_name }) open(self.GMETAD_CONF, 'w').write(str(tmpl)) # Frontend configuration if not os.path.isdir('/var/www'): os.mkdir('/var/www') run_cmd('cp -a /root/ConPaaS/contrib/ganglia_frontend/ganglia /var/www') copy(os.path.join(self.cps_home, 'contrib', 'ganglia_modules', 'nginx-manager.conf'), '/var/cache/cpsagent') copy('/etc/nginx/fastcgi_params', '/var/cache/cpsagent/') copy(os.path.join(self.cps_home, 'contrib', 'ganglia_modules', 'www.conf'), '/etc/php5/fpm/pool.d/') copyfile(os.path.join(self.cps_home, 'config', 'ganglia', 'ganglia_frontend.tmpl'), '/etc/nginx/nginx.conf')
def configure(self): """Here we also need to configure gmetad and the ganglia frontend""" BaseGanglia.configure(self) # Write gmetad.conf src = open(os.path.join(self.cps_home, 'config', 'ganglia', 'ganglia-gmetad.tmpl')).read() tmpl = Template(src, { 'clusterName': self.cluster_name }) open(self.GMETAD_CONF, 'w').write(str(tmpl)) # Frontend configuration if not os.path.isdir('/var/www'): os.mkdir('/var/www') run_cmd('cp -a /root/ConPaaS/contrib/ganglia_frontend/ganglia /var/www') copy(os.path.join(self.cps_home, 'contrib', 'ganglia_modules', 'nginx-manager.conf'), '/var/cache/cpsagent') copy('/etc/nginx/fastcgi_params', '/var/cache/cpsagent/') copy(os.path.join(self.cps_home, 'contrib', 'ganglia_modules', 'www.conf'), '/etc/php5/fpm/pool.d/') copyfile(os.path.join(self.cps_home, 'config', 'ganglia', 'ganglia_frontend.tmpl'), '/etc/nginx/nginx.conf') if 'php' in self.service_type: modules = ['num_machines_provisioning'] BaseGanglia.add_modules(self, modules )
def configure(self): """Here we also need to configure gmetad and the ganglia frontend""" BaseGanglia.configure(self) # Write gmetad.conf src = open(os.path.join(self.cps_home, "config", "ganglia", "ganglia-gmetad.tmpl")).read() tmpl = Template(src, {"clusterName": self.cluster_name}) open(self.GMETAD_CONF, "w").write(str(tmpl)) # Frontend configuration if not os.path.isdir("/var/www"): os.mkdir("/var/www") run_cmd("cp -a /root/ConPaaS/contrib/ganglia_frontend/ganglia /var/www") copy(os.path.join(self.cps_home, "contrib", "ganglia_modules", "nginx-manager.conf"), "/var/cache/cpsagent") copy("/etc/nginx/fastcgi_params", "/var/cache/cpsagent/") copy(os.path.join(self.cps_home, "contrib", "ganglia_modules", "www.conf"), "/etc/php5/fpm/pool.d/") copyfile(os.path.join(self.cps_home, "config", "ganglia", "ganglia_frontend.tmpl"), "/etc/nginx/nginx.conf") if "php" in self.service_type: modules = ["num_machines_provisioning"] BaseGanglia.add_modules(self, modules)
def _mount(self, dev_name, mount_point, mkfs): devnull_fd = open(devnull, 'w') # waiting for our block device to be available dev_found = False dev_prefix = dev_name.split('/')[2][:-1] for attempt in range(1, 11): self.logger.info("Generic node waiting for block device '%s'" % dev_name) if self._check_dev_is_attached(dev_name): dev_found = True break else: # On EC2 the device name gets changed # from /dev/sd[a-z] to /dev/xvd[a-z] if self._check_dev_is_attached(dev_name.replace(dev_prefix, 'xvd')): dev_found = True dev_name = dev_name.replace(dev_prefix, 'xvd') self.logger.info("Block device is renamed to '%s'" % dev_name) break time.sleep(10) # create mount point mkdir_cmd = "mkdir -p %s" % mount_point run_cmd(mkdir_cmd) if dev_found: self.logger.info("Generic node has now access to '%s'" % dev_name) # prepare block device if mkfs: self.logger.info("Creating new file system on '%s'" % dev_name) prepare_args = ['mkfs.ext4', '-q', '-m0', dev_name] proc = Popen(prepare_args, stdin=PIPE, stdout=devnull_fd, stderr=devnull_fd, close_fds=True) proc.communicate(input="y") # answer interactive question with y if proc.wait() != 0: self.logger.critical('Failed to prepare storage device:(code=%d)' % proc.returncode) else: self.logger.info('File system created successfully') else: self.logger.info( "Not creating a new file system on '%s'" % dev_name) time.sleep(10) # mount mount_args = ['mount', dev_name, mount_point] mount_cmd = ' '.join(mount_args) self.logger.debug("Running command '%s'" % mount_cmd) _, err = run_cmd(mount_cmd) if err: self.logger.critical('Failed to mount storage device: %s' % err) else: self.logger.info("Generic node has prepared and mounted '%s'" % mount_point) else: self.logger.critical("Block device '%s' unavailable" % dev_name)
def mount(self, mkfs): self.state = S_STARTING devnull_fd = open(devnull,'w') # waiting for our block device to be available dev_found = False dev_prefix = self.dev_name.split('/')[2][:-1] for attempt in range(1, 11): sql_logger.info("Galera node waiting for block device %s" % self.dev_name) if lexists(self.dev_name): dev_found = True break else: # On EC2 the device name gets changed # from /dev/sd[a-z] to /dev/xvd[a-z] if lexists(self.dev_name.replace(dev_prefix, 'xvd')): dev_found = True self.dev_name = self.dev_name.replace(dev_prefix, 'xvd') break time.sleep(10) # create mount point run_cmd(self.mkdir_cmd) if dev_found: sql_logger.info("Galera node has now access to %s" % self.dev_name) # prepare block device if mkfs: sql_logger.info("Creating new file system on %s" % self.dev_name) self.prepare_args = ['mkfs.ext4', '-q', '-m0', self.dev_name] proc = Popen(self.prepare_args, stdin=PIPE, stdout=devnull_fd, stderr=devnull_fd, close_fds=True) proc.communicate(input="y") # answer interactive question with y if proc.wait() != 0: sql_logger.critical('Failed to prepare storage device:(code=%d)' % proc.returncode) else: sql_logger.info('File system created successfully') else: sql_logger.info( "Not creating a new file system on %s" % self.dev_name) time.sleep(10) # mount self.mount_args = ['mount', self.dev_name, self.mount_point] mount_cmd = ' '.join(self.mount_args) sql_logger.debug("Running command '%s'" % mount_cmd) _, err = run_cmd(mount_cmd) if err: sql_logger.critical('Failed to mount storage device: %s' % err) else: sql_logger.info("OSD node has prepared and mounted %s" % self.dev_name) else: sql_logger.critical("Block device %s unavailable, falling back to image space" % self.dev_name)
def add_modules(self, modules): """Install additional modules and restart ganglia-monitor""" for module in modules: # Copy conf files into ganglia conf.d filename = os.path.join(self.cps_home, "contrib", "ganglia_modules", module + ".pyconf") copy(filename, os.path.join(self.GANGLIA_CONFD, module + ".conf")) # Copy python modules filename = os.path.join(self.cps_home, "contrib", "ganglia_modules", module + ".py") copy(filename, self.GANGLIA_MODULES_DIR) # Restart ganglia-monitor run_cmd("/etc/init.d/ganglia-monitor restart")
def _check_dev_is_attached(self, dev_name): # if the device file does not exist, the volume is definitely not # attached yet if not lexists(dev_name): return False # force the kernel to re-read the partition table # this allows reusing the device name after a volume was detached run_cmd('sfdisk -R %s' % dev_name) # check if the device appears in the partitions list short_dev_name = dev_name.split('/')[2] output, _ = run_cmd('cat /proc/partitions') return short_dev_name in output
def add_modules(self, modules): """Install additional modules and restart ganglia-monitor""" for module in modules: # Copy conf files into ganglia conf.d filename = os.path.join(self.cps_home, 'contrib', 'ganglia_modules', module + '.pyconf') copy(filename, os.path.join(self.GANGLIA_CONFD, module + '.conf')) # Copy python modules filename = os.path.join(self.cps_home, 'contrib', 'ganglia_modules', module + '.py') copy(filename, self.GANGLIA_MODULES_DIR) # Restart ganglia-monitor run_cmd('/etc/init.d/ganglia-monitor restart')
def get_ip_address(): """Return the IPv4 address of this IPOP node and its routing prefix in CIDR notation. Empty string if it is not set. eg: get_ip_address() -> '192.168.12.42/24' """ ip_cmd = "ip -f inet addr show dev tapipop | grep inet | awk '{ print $2 }'" return run_cmd(ip_cmd, '/')[0].rstrip('\n')
def unmount(self): # kill all processes still using the volume sql_logger.info("Killing all processes using the Galera Disk") fuser_args = ['fuser', '-km', self.dev_name] fuser_cmd = ' '.join(fuser_args) sql_logger.debug("Running command '%s'" % fuser_cmd) run_cmd(fuser_cmd) # unmount sql_logger.info("Trying to unmount the Galera Disk") unmount_args = ['umount', self.dev_name] unmount_cmd = ' '.join(unmount_args) sql_logger.debug("Running command '%s'" % unmount_cmd) _, err = run_cmd(unmount_cmd) if err: sql_logger.critical('Failed to unmount storage device: %s' % err) else: sql_logger.info("Galera node has succesfully unmounted %s" % self.dev_name)
def _unmount(self, vol_name): mount_point = join(self.VOLUME_DIR, vol_name) # kill all processes still using the volume fuser_args = ['fuser', '-km', mount_point] fuser_cmd = ' '.join(fuser_args) self.logger.debug("Running command '%s'" % fuser_cmd) run_cmd(fuser_cmd) # unmount unmount_args = ['umount', mount_point] unmount_cmd = ' '.join(unmount_args) self.logger.debug("Running command '%s'" % unmount_cmd) _, err = run_cmd(unmount_cmd) if err: self.logger.critical('Failed to unmount storage device: %s' % err) else: self.logger.info("Generic node has succesfully unmounted '%s'" % mount_point)
def git_code_version(repo): """Get the abbreviated commit hash of the last commit performed in the specified repository. The return value is a string matching the following regular expression: /^[a-z0-9]{7,7}$/ """ cmd = 'git log -1 --pretty=oneline --format="%h"' return run_cmd(cmd, repo)[0].rstrip()
def git_enable_revision(target_dir, repo, rev, subdir=None): """This function clones a local git repository and performs a git-checkout of the specified revision. The contents are moved to 'target_dir'. If the 'subdir' parameter is present, only its contents are moved to 'target_dir'. """ temp_dir = tempfile.mkdtemp() run_cmd(cmd='git clone %s %s' % (repo, temp_dir)) run_cmd(cmd='git checkout %s' % rev, directory=temp_dir) if subdir is None: os.rename(temp_dir, target_dir) else: temp_sub_dir = os.path.join(temp_dir, subdir) if os.path.isdir(temp_sub_dir): os.rename(temp_sub_dir, target_dir) else: os.makedirs(target_dir) shutil.rmtree(temp_dir)
def get_snapshot(self, kwargs): # pack data from: # /etc/xos/xtreemfs/ (some files later for certificates) # /var/lib/xtreemfs/ (everything, after shutting down the services) filename = "/root/snapshot.tar.gz" dirs = "var/lib/xtreemfs/ etc/xos/xtreemfs/ var/log/xtreemfs/" err, out = run_cmd("tar -czf %s %s" % (filename, dirs), "/") if err: self.logger.exception(err) return HttpErrorResponse(err) return HttpFileDownloadResponse("snapshot.tar.gz", filename)
def sqldump(self, kwargs): if self.state != self.S_RUNNING: return HttpErrorResponse('ERROR: Wrong state to call sqldump') master_ip = self.config.getMySQLmasters()[0].ip cmd = 'mysqldump -u mysqldb -h %s --password=%s -A' % (master_ip, self.root_pass) out, err = run_cmd(cmd) if err: return HttpErrorResponse(err) return HttpJsonResponse(out)
def start(self): """We also need to start gmetad, php5-fpm and nginx""" err = BaseGanglia.start(self) if err: return err cmds = ('/etc/init.d/gmetad start', '/etc/init.d/php5-fpm start', '/usr/sbin/nginx -c /var/cache/cpsagent/nginx-manager.conf') for cmd in cmds: _, err = run_cmd(cmd) if err: return "Error executing '%s': %s" % (cmd, err)
def git_enable_revision(target_dir, repo, rev): """This function clones a local git repository into target_dir/rev. It then performs a git-checkout of the specified revision. As an example, the following invocation: git_enable_revision("/var/www", "/home/git/code", "8d3b8a6") is roughly equivalent to these shell commands: git clone /home/git/code /var/www/8d3b8a6 cd /var/www/8d3b8a6 git checkout 8d3b8a6 As a result, a directory is created containing the specified revision of the given repository, and its absolute pathname is returned as a string. """ dest_dir = os.path.join(target_dir, rev) run_cmd(cmd='git clone %s %s' % (repo, rev), directory=target_dir) run_cmd(cmd='git checkout %s' % rev, directory=dest_dir) return dest_dir
def git_create_tmp_repo(): """Create a temporary git repository with a pseudo-random name. Return the repository absolute path as a string. This function is useful for automated testing purposes.""" repo_path = tempfile.mkdtemp() cmd = 'git init %s' % repo_path run_cmd(cmd, repo_path) cmd = 'git add %s' % tempfile.mkstemp(dir=repo_path)[1] run_cmd(cmd, repo_path) cmd = 'git config user.email "*****@*****.**"' run_cmd(cmd, repo_path) cmd = 'git config user.name "ConPaaS"' run_cmd(cmd, repo_path) cmd = 'git commit -am "Initial commit"' run_cmd(cmd, repo_path) return repo_path
def start(self): """We also need to start gmetad, php5-fpm and nginx""" err = BaseGanglia.start(self) if err: return err cmds = ( '/etc/init.d/gmetad start', '/etc/init.d/php5-fpm start', '/usr/sbin/nginx -c /var/cache/cpsagent/nginx-manager.conf' ) for cmd in cmds: _, err = run_cmd(cmd) if err: return "Error executing '%s': %s" % (cmd, err)
def get_snapshot(self, kwargs): # pack data from: # /etc/cpsagent/certs/ (original agent certificates) # /etc/xos/xtreemfs/ (config files, SSL-certificates, policies) # /var/lib/xtreemfs/ (save everything, after shutting down the services) # /var/log/xtreemfs/ (xtreemfs log files) filename = "/root/snapshot.tar.gz" dirs = "var/lib/xtreemfs/ etc/xos/xtreemfs/ var/log/xtreemfs/ etc/cpsagent/certs" err, out = run_cmd("tar -czf %s %s" % (filename, dirs), "/") if err: self.logger.exception(err) return HttpErrorResponse(err) return HttpFileDownloadResponse("snapshot.tar.gz", filename)
def set_snapshot(self, kwargs): if 'archive_url' not in kwargs: raise AgentException(AgentException.E_ARGS_MISSING, 'archive_url') self.logger.info('set_snapshot: restoring archive') archive_cmd = "wget --ca-certificate /etc/cpsagent/certs/ca_cert.pem " archive_cmd += kwargs['archive_url'] + " -O - | tar xz -C /" self.logger.debug('set_snapshot: %s' % archive_cmd) out, err = run_cmd(archive_cmd) self.logger.debug('set_snapshot: stdout %s' % out) self.logger.debug('set_snapshot: stderr %s' % err) self.logger.info('set_snapshot: archive restored successfully') return HttpJsonResponse()
def _create_user_cert(self, user, group, passphrase, adminflag): # create a temporary directory self.logger.debug('_create_user_cert: creating tmp dir') tmpdir = tempfile.mkdtemp() self.logger.debug('_create_user_cert: created tmp dir') # create certificates and truststore creation_cmd = "bash " creation_cmd += self.config_parser.get('manager', 'CONPAAS_HOME') creation_cmd += "/src/conpaas/services/xtreemfs/etc/" creation_cmd += "generate_certificate.sh -u " + user + " -g " + group + " -p " + passphrase if adminflag: creation_cmd += " -s" self.logger.debug('_create_user_cert: executing script') out, err = run_cmd(creation_cmd, tmpdir) self.logger.debug('_create_user_cert: stderr %s' % out) self.logger.debug('_create_user_cert: stdout %s' % err) # store result as base64 encoded string into dictionary for transfer return open(tmpdir + "/" + user + ".p12", 'rb').read()
def set_snapshot(self, kwargs): if 'archive_url' not in kwargs: raise AgentException( AgentException.E_ARGS_MISSING, 'archive_url') self.logger.info('set_snapshot: restoring archive') archive_cmd = "wget --ca-certificate /etc/cpsagent/certs/ca_cert.pem " archive_cmd += kwargs['archive_url'] + " -O - | tar xz -C /" self.logger.debug('set_snapshot: %s' % archive_cmd) out, err = run_cmd(archive_cmd) self.logger.debug('set_snapshot: stdout %s' % out) self.logger.debug('set_snapshot: stderr %s' % err) self.logger.info('set_snapshot: archive restored successfully') return HttpJsonResponse()
def _create_certs(self, nodes): for node in nodes: certs = {} # create a temporary directory tmpdir = tempfile.mkdtemp() # create certificates and truststore creation_cmd = "bash " creation_cmd += self.config_parser.get('manager', 'CONPAAS_HOME') creation_cmd += "/src/conpaas/services/xtreemfs/etc/" creation_cmd += "generate_certificate.sh -dmo -t" out, err = run_cmd(creation_cmd, tmpdir) self.logger.debug('_create_certs: stderr %s' % out) self.logger.debug('_create_certs: stdout %s' % err) # store result as base64 encoded string into dictionary for transfer certs['dir'] = base64.b64encode(open(tmpdir + "/dir.p12", 'rb').read()) certs['mrc'] = base64.b64encode(open(tmpdir + "/mrc.p12", 'rb').read()) certs['osd'] = base64.b64encode(open(tmpdir + "/osd.p12", 'rb').read()) certs['truststore'] = base64.b64encode(open(tmpdir + "/trusted.jks", 'rb').read()) # transfer data to agent node client.set_certificates(node.ip, 5555, certs)
def set_snapshot(self, kwargs): try: exp_params = [('archive_url', is_string)] archive_url = check_arguments(exp_params, kwargs) except Exception as ex: return HttpErrorResponse("%s" % ex) self.logger.info('set_snapshot: restoring archive') archive_cmd = "curl --cacert /etc/cpsagent/certs/ca_cert.pem " archive_cmd += archive_url + " | tar xz -C /" self.logger.debug('set_snapshot: %s' % archive_cmd) out, err = run_cmd(archive_cmd) self.logger.debug('set_snapshot: stdout %s' % out) self.logger.debug('set_snapshot: stderr %s' % err) self.logger.info('set_snapshot: archive restored successfully') return HttpJsonResponse()
def restart_ipop(): """Restart the IPOP service.""" return run_cmd(cmd='/etc/init.d/groupvpn.sh restart', directory='/')
def __init__(self, config, nodes, device_name): """ Creates a configuration from `config`. :param config: Configuration read with ConfigParser. :type config: ConfigParser :param _dummy_config: Set to `True` when used in unit testing. :type boolean: boolean value. """ sql_logger.debug("Trying to Mount the device.") try: # Mount device self.dev_name = "/dev/%s" % device_name self.mount_point = "/media/GaleraDisk" self.mkdir_cmd = "mkdir -p %s" % self.mount_point self.mount(True) except ConfigParser.Error: sql_logger.exception('Could not mount the device') sql_logger.debug("Entering init MySQLServerConfiguration") try: #hostname = socket.gethostname() # Get connection settings #self.pid_file = "/var/lib/mysql/" + hostname + ".pid" sql_logger.debug("Trying to get params from configuration file ") self.conn_location = config.get('MySQL_root_connection', 'location') self.conn_username = config.get("MySQL_root_connection", "username") self.conn_password = config.get("MySQL_root_connection", "password") sql_logger.debug("Got parameters for root connection to MySQL") # Get MySQL configuration parameters self.mysqldump_path = config.get('agent', 'MYSQLDUMP_PATH') self.mycnf_filepath = config.get("MySQL_configuration", "my_cnf_file") self.path_mysql_ssr = config.get("MySQL_configuration", "path_mysql_ssr") # We need to remove the bind-address option (the default one is # 127.0.0.1 = accepting connections only on local interface) #ConfigParser can't be used for my.cnf because of options without value, thus: f = open(self.mycnf_filepath, "r") lines = f.readlines() f.close() f = open(self.mycnf_filepath, "w") #Creating the MySQL directory tree in the external disk sql_logger.debug("Creating the MySQL directory tree in the external disk") run_cmd("mkdir -p %s/data/" % self.mount_point) run_cmd("mkdir -p %s/tmp/" % self.mount_point) run_cmd("cp -a /var/lib/mysql/* %s/data/" % self.mount_point) run_cmd("chown -R mysql:mysql %s/" % self.mount_point) for line in lines: if "datadir" in line: f.write("datadir=%s/data/\n" % self.mount_point) #elif "socket" in line: # f.write("socket=%s/data/\n" % self.mount_point) elif not "bind-address" in line: f.write(line) if "expire_logs_days" in line: f.write("log_error=/root/mysql.log\n" ) f.close() self.wsrep_filepath = config.get("Galera_configuration", "wsrep_file") self.wsrep_user = config.get("Galera_configuration", "wsrep_sst_username") self.wsrep_password = config.get("Galera_configuration", "wsrep_sst_password") self.wsrep_provider = config.get("Galera_configuration", "wsrep_provider") self.wsrep_sst_method = config.get("Galera_configuration", "wsrep_sst_method") self.glbd_location = config.get("Galera_configuration", "glbd_location") self.wsrepconfig = ConfigParser.ConfigParser() self.wsrepconfig.optionxform = str # to preserve case in option names self.wsrepconfig.read(self.wsrep_filepath) path = self.wsrep_filepath cluster_nodes = ','.join(nodes) self.wsrepconfig.set("mysqld", "wsrep_cluster_address", "\"gcomm://%s\"" % cluster_nodes) self.wsrepconfig.set("mysqld", "wsrep_provider", self.wsrep_provider) self.wsrepconfig.set("mysqld", "wsrep_sst_method", self.wsrep_sst_method) self.wsrepconfig.set("mysqld", "wsrep_sst_auth", "%s:%s" % (self.wsrep_user, self.wsrep_password)) os.remove(path) newf = open(path, "w") self.wsrepconfig.write(newf) newf.close() self.state = S_STOPPED # Start self.start() # Change root password when starting is_first_node = len(nodes) == 0 if is_first_node: os.system("mysqladmin -u root password " + self.conn_password) #TODO: add user conn_userame and grant privileges to it from any host self.add_user(self.conn_username, self.conn_password) self.add_user(self.wsrep_user, self.wsrep_password) except ConfigParser.Error: sql_logger.exception('Could not read config file') except IOError: sql_logger.exception('Config file doesn\'t exist') # Creating a supervisor object sql_logger.debug("Leaving init MySQLServerConfiguration")
def start(self): """Services startup""" _, err = run_cmd('/etc/init.d/ganglia-monitor start') if err: return 'Error starting ganglia-monitor: %s' % err
def start(self, mkfs): self.state = S_STARTING devnull_fd = open(devnull, 'w') # waiting for our block device to be available dev_found = False for attempt in range(1, 11): logger.info("OSD node waiting for block device %s" % self.dev_name) if lexists(self.dev_name): dev_found = True break else: # On EC2 the device name gets changed # from /dev/sd[a-z] to /dev/xvd[a-z] self.dev_name = self.dev_name.replace('sd', 'xvd') if lexists(self.dev_name): dev_found = True break else: self.dev_name = self.dev_name.replace('xvd', 'sd') time.sleep(10) # create mount point run_cmd(self.mkdir_cmd) if dev_found: logger.info("OSD node has now access to %s" % self.dev_name) # prepare block device if mkfs: logger.info("Creating new file system on %s" % self.dev_name) self.prepare_args = ['mkfs.ext4', '-q', '-m0', self.dev_name] proc = Popen(self.prepare_args, stdin=PIPE, stdout=devnull_fd, stderr=devnull_fd, close_fds=True) proc.communicate( input="y") # answer interactive question with y if proc.wait() != 0: logger.critical( 'Failed to prepare storage device:(code=%d)' % proc.returncode) else: logger.info('File system created successfully') else: logger.info("Not creating a new file system on %s" % self.dev_name) time.sleep(10) # mount self.mount_args = ['mount', self.dev_name, self.mount_point] mount_cmd = ' '.join(self.mount_args) logger.debug('Running command %s' % mount_cmd) _, err = run_cmd(mount_cmd) if err: logger.critical('Failed to mount storage device: %s' % err) else: logger.info("OSD node has prepared and mounted %s" % self.dev_name) else: logger.critical( "Block device %s unavailable, falling back to image space" % self.dev_name) # must be owned by xtreemfs:xtreemfs xtreemfs_user = getpwnam("xtreemfs") chown(self.mount_point, xtreemfs_user.pw_uid, xtreemfs_user.pw_gid) logger.debug(str(self.start_args)) devnull_fd = open(devnull, 'w') proc = Popen(self.start_args, stdout=devnull_fd, stderr=devnull_fd, close_fds=True) if proc.wait() != 0: logger.critical('Failed to start osd service:(code=%d)' % proc.returncode) logger.info('OSD Service started.') self.state = S_RUNNING
def git_push(repo, destination_ip): """Push repository contents to the given destination_ip""" cmd = 'git push git@%s:%s master' % (destination_ip, DEFAULT_CODE_REPO) return run_cmd(cmd, repo)
# Regardless of node type, restore metadata try: self.logger.info('set_service_snapshot: restoring %s' % node.ip) data = client.set_snapshot(node.ip, 5555, data['archive']) except client.AgentException, err: self.logger.exception(err) raise err # restore manager data manager_data = kwargs['manager'] filename = "/root/manager_data.tar.gz" open(filename, 'wb').write(base64.b64decode(manager_data)) err, out = run_cmd("tar -xzf %s" % filename, "/") if err: self.logger.exception(err) return HttpErrorResponse(err) self.logger.info("set_service_snapshot: starting all agent services") self._start_all() self.logger.info("set_service_snapshot: all agent services started") return HttpJsonResponse() @expose('POST') def get_user_cert(self, kwargs): if not 'user' in kwargs: return HttpErrorResponse("ERROR: Required argument (user) doesn't exist") if not 'group' in kwargs:
def git_last_description(repo): """Return a string containing the subject line of the last commit performed in the specified repository.""" cmd = 'git log -1 --pretty=oneline --format="%s"' return run_cmd(cmd, repo)[0].rstrip()
def start(self): """Services startup""" _, err = run_cmd("/etc/init.d/ganglia-monitor start") if err: return "Error starting ganglia-monitor: %s" % err
def get_service_snapshot(self, kwargs): if self.state != self.S_RUNNING: return HttpErrorResponse( 'ERROR: Wrong state to get service snapshot.') self.state = self.S_EPILOGUE # stop all agent services self.logger.debug("Stopping all agent services") self._stop_all(remove=False) self.logger.debug("Calling get_snapshot on agents") # dictionary mapping node IDs to tuples of uuids/None (DIR, MRC, OSD) nodes_snapshot = {} for node in self.nodes: if node.id not in nodes_snapshot: nodes_snapshot[node.id] = { 'data': None, 'dir_uuid': self.dir_node_uuid_map.get(node.id), 'mrc_uuid': self.mrc_node_uuid_map.get(node.id), 'osd_uuid': self.osd_node_uuid_map.get(node.id) } try: # get snapshot from this agent node, independent of what # XtreemFS services are running there data = client.get_snapshot(node.ip, 5555) self.logger.debug('get_snapshot(%s) HTTP code: %s' % (node.ip, data[0])) nodes_snapshot[node.id]['data'] = base64.b64encode(data[1]) except client.AgentException: self.logger.exception('Failed to get snapshot from node %s' % node) self.state = self.S_ERROR raise # Get ID of attached volume volume_id = self.osd_uuid_volume_map.get( nodes_snapshot[node.id]['osd_uuid']) nodes_snapshot[node.id]['volume'] = volume_id if volume_id: volume = self.get_volume(volume_id) nodes_snapshot[node.id]['cloud'] = volume.cloud.cloud_name for key in 'dir_uuid', 'mrc_uuid', 'osd_uuid', 'volume': self.logger.debug("nodes_snapshot[%s]['%s']: %s" % (node.id, key, nodes_snapshot[node.id][key])) # manager data filename = "/root/snapshot.tar.gz" dirs = self.config_parser.get('manager', 'CERT_DIR') err, out = run_cmd("tar -czf %s %s" % (filename, dirs), "/") if err: self.logger.exception(err) return HttpErrorResponse(err) manager_data = base64.b64encode(open(filename, 'rb').read()) self.logger.debug("Shutting all agents down") self._do_shutdown(stop_services=False) return HttpJsonResponse({'nodes' : nodes_snapshot.values(), 'manager' : manager_data})