def check_for_gems(gem): """ Check that a gem exists. Before checking for gem, runs `type gem` command to check that gem commands are available. Outputs log as it checks for gem. Args: gem(str): gem to search for Returns: returncode (int) -- 0 if successful find, 1 otherwise (either not found or gem command unavailable) Example Usage: >>> print check_for_gems("bundler") 0 """ slab_logger.log(15, 'Checking for gem %s' % gem) returncode, myinfo = service_utils.run_this("type gem") if returncode == 0: returncode_b, myinfo_b = service_utils.run_this( 'gem list | grep -q %s' % (gem)) if returncode_b == 1: slab_logger.error("Cannot find gem(%s): %s" % (gem, myinfo_b)) return (returncode_b) else: return (returncode_b) else: slab_logger.error("Cannot find gem command: %s" % (myinfo)) return (returncode)
def setup_ruby(username=None): """ Ruby set up. Installs rvm Sets username if none set. Args: username (str): defaults to None Returns: Returns no variables. Example Usage: >>> print setup_ruby() #rvm installed #username is set to aaltman """ slab_logger.log(15, 'Installing rvm') # Note: I needed this on my CentOS7 machine, but not my MAC. # Being allowed to fail for now. service_utils.run_this("gpg2 --keyserver hkp://keys.gnupg.net \ --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3") returncode = service_utils.run_this("curl -L get.rvm.io | bash -s stable") if returncode > 0: slab_logger.error("Failed to install RVM.") return (1) else: if username: pass else: returncode, username = helper_utils.get_gitusername(ctx.path) if returncode > 0: slab_logger.error("Couldn't set user.")
def __init__( self, vm_name, path, provider="virtualbox", default_vbox="Cisco/rhel-7", default_vbox_url='http://cis-kickstart.cisco.com/ccs-rhel-7.box'): """ Constructor for Vagrant Class Assigns member variables to constructor parameters. Sets up Vagrant Client. Args: self -- self-referencing pointer vname -- name of the virtual machine path -- The path to your working .stack directory. Typically, this looks like ./servicelab/servicelab/.stack where "." is the path to the root of the servicelab repository. provider -- vagrant provider, defaults to virtualbox default_vbox -- default virtualbox, defaults to "Cisco/Rhel7" default_vbox_url -- hyperlink to the default_vbox """ self.vm_name = vm_name self.provider = provider self.default_vbox = default_vbox self.default_vbox_url = default_vbox_url self.path = path # Note: The quiet is so we know what's happening during # vagrant commands in the term. # Note: Setup vagrant client. vagrant_dir = path self.v = vagrant.Vagrant(root=vagrant_dir, quiet_stdout=False, quiet_stderr=False) # check for installed plugins # over here we will install the plugins. # this is an extra step but by this we can avoid extra command repitition # the other alternative is to put it in vagrantfile but this can be a problem # as we may have to exec over the running process # please see stack discussion # http://stackoverflow.com/questions/19492738/demand-a-vagrant-plugin-within-the-vagrantfile req_lst = ["vagrant-hostmanager", "vagrant-openstack-provider"] plugin_lst = self.v.plugin_list() for plugin in req_lst: if plugin not in plugin_lst: service_utils.run_this( 'vagrant plugin install {}'.format(plugin))
def create_repo(self): """ create a repo on the local machine. Returns -1 if the repo creation fails. Returns: ( int : return code str : output string if any. ) """ slab_logger.log(15, 'Creating local repo %s' % self.get_reponame()) hostname = self.gsrvr['hostname'] port = self.gsrvr['port'] # please see https://code.google.com/p/gerrit/issues/detail?id=1013 # for -no-checkout option. cmd = "git clone --no-checkout --depth=1 " cmd += "ssh://{}@{}:{}/{} {}/.tmp".format(self.username, hostname, port, self.get_reponame(), self.get_reponame()) ret_code, ret_str = service_utils.run_this(cmd) assert ret_code == 0, "unable to clone the project:" + ret_str os.rename(os.path.join(self.get_reponame(), ".tmp", ".git"), os.path.join(self.get_reponame(), ".git")) shutil.rmtree(os.path.join(self.get_reponame(), ".tmp")) return ret_code
def encrypt(pub_fname, data): """Encrypts data using the public key. Args: pub_fname (str): Public Key data (str): data that will be encrypted by the public key Returns: returncode (int): 0 -- Success 1 -- Failure, can't create key (possibly invalid certificate) cmd_info (str): stdout/stderr log of pubkey-creation command Example Usage: >>> print encrypt(public_key_pem) (0, "") #Data encryption is also outputted """ slab_logger.log(15, 'Encrpyting data using ssl public key') code = "\"" code = code + "require 'openssl'; require 'base64';" code = code + "public_key_pem = File.read '" + pub_fname + "';" code = code + "public_key = OpenSSL::X509::Certificate.new(public_key_pem);" code = code + "cipher = OpenSSL::Cipher::AES.new(256, :CBC);" code = code + "enc = OpenSSL::PKCS7::encrypt([public_key], '" code = code + data + "', " + "cipher, OpenSSL::PKCS7::BINARY).to_der;" code = code + "print Base64.encode64(enc).gsub(/\n/, '');" code = code + "\"" cmd = "ruby -e " + code cmd_returncode, cmd_info = service_utils.run_this(cmd) if cmd_returncode > 0: slab_logger.error(cmd_info) return (cmd_returncode, cmd_info)
def compile_man_page(path, user, password): """ Grabs data from confluence pages and servicelab docs and leverages sphinx to convert them into a single man page that will be queried for high level info. """ slab_logger.log(15, 'Building data for man pages') path_to_docs = os.path.split(path)[0] path_to_docs = os.path.split(path_to_docs)[0] path_to_docs = os.path.join(path_to_docs, 'docs') man_yaml = _load_slabmanyaml(path) # The man_contents.rst.inc file follows Sphinx syntax for enumerating the # files that will be included in the final man page with open(os.path.join(path_to_docs, "man_contents.rst.inc"), "w") as manf: manf.write(".. toctree::\n :maxdepth: 2\n\n") manf.writelines(" " + elem + "\n" for elem in man_yaml["slab_docs"]) # Manually download all desired sites, and leverage Sphinx's "make man" command # to build a single man page using pages indexed above for item in man_yaml['confluence_pages']: url = 'https://confluence.sco.cisco.com/display/' + item site_title = item.translate(None, '+') file_title = item.translate(maketrans("+", "-")).lower()[4:] content = requests.get(url, auth=(user, password)) if content.status_code != 200: slab_logger.error( "Unable to login to %s as user %s with supplied password." % (url, user)) return with open( os.path.join(path_to_docs, 'man_pages', '%s.rst' % file_title), 'w') as manf: manf.write(_parse_confluence_website(content.text, site_title)) service_utils.run_this( "echo ' man_pages/%s' >> man_contents.rst.inc" % file_title, path_to_docs) cmd = "sed -i -e \"s/master_doc = 'index'/master_doc = 'man_index'/g\" conf.py; " \ "make man;" \ "sed -i -e \"s/master_doc = 'man_index'/master_doc = 'index'/g\" conf.py;" service_utils.run_this(cmd, path_to_docs)
def download_template(self): """ Download the created project from the gerrit server """ hostname = self.gsrvr['hostname'] port = self.gsrvr['port'] cmd = "git clone --depth=1 " cmd += "ssh://{}@{}:{}/{}".format(self.username, hostname, port, self.get_reponame()) ret_code, ret_str = service_utils.run_this(cmd) assert ret_code == 0, "unable to get project template project:" + ret_str
def repo_list(self): """ Generate list of all repos on gerrit server.""" slab_logger.debug('Generating list of all repos on gerrit') key = self.getkey() cmd = "ssh -i {} -p {} {}@{} gerrit ls-projects".format( key, self.port, self.user, self.hostname) # now run the command and get output ret_code, ret_str = service_utils.run_this(cmd) if ret_code: raise GerritFnException(ret_str) return ret_str.split()
def download_template(self): """ Download the service-helloworld-ansible template from the gerrit server """ slab_logger.log(15, 'Downloading the service-helloworld-ansible template') hostname = self.gsrvr['hostname'] port = self.gsrvr['port'] cmd = "git clone --depth=1 " cmd += "ssh://{}@{}:{}/service-helloworld-puppet {}".format( self.username, hostname, port, self.get_reponame()) ret_code, ret_str = service_utils.run_this(cmd) assert ret_code == 0, "unable to get puppet template project:" + ret_str
def copy_file_to_vm(file_name, vm_name, file_path, port_no, vm_path, path, key_path): """ Gets ssh port for vm Args: vm_name path to Vagrantfile Returns: return_code, command output Example Usage: my_class_var.copy_file_to_vm(file_path, port_no, vm_path, path, key_path) """ cmd_string = "scp -i %s -P %s %s [email protected]:%s" cmd = cmd_string % (key_path, port_no, file_path, "/tmp") service_utils.run_this(cmd, path) cmd_string = "vagrant ssh %s -c \"sudo cp /tmp/%s %s\"" cmd = cmd_string % (vm_name, file_name, vm_path) return service_utils.run_this(cmd, path)
def get_ruby_version(): """ Returns the version of Ruby. Returns: Returns ruby version # or nothing if command `ruby -v` fails. Example Usage: >>> print get_ruby_version() 2.1.6 """ slab_logger.log(15, 'Determining ruby version') returncode, cmd_info = service_utils.run_this('ruby -v') if returncode != 0: return None match = re.search("[0-9]+.[0-9]+.[0-9]+", cmd_info) return match.group(0)
def _get_number_of_matches(section_name, path_to_man_page, query_str): """ Return the number of times query is found in the man page section """ slab_logger.log(15, 'Determining number of query matches') # Equivalent to _view_section, except for the added grep command which # ignores case and outputs matches, which are then counted by wc section_name1 = section_name.replace(' ', r'\ ') section_name2 = "^" + ".*".join(section_name) + "$" cmd_view = "man -P 'less -p ^%s' %s | " % (section_name1, path_to_man_page) cmd_view += "sed -n -e '/%s/,/^[^ \\t\\r\\n\\v\\f]/ p' | " % ( section_name2) cmd_view += "sed -e '$d' | grep -io %s | wc -l " % (query_str) # ignoring the error. just printing the output. this should be reviewed _, output = service_utils.run_this(cmd_view) return int(output.strip())
def get_ssh_key_path_for_vm(vm_name, path): """ Gets ssh key for vm Args: vm_name path to Vagrantfile Returns: return_code, command output (key path) Example Usage: my_class_var.get_ssh_key_path_for_vm(vm_name, path) """ cmd = "vagrant ssh-config | grep %s | grep IdentityFile"\ " | awk '{print $2}'" % (vm_name) return_code, key_path = service_utils.run_this(cmd, path) return return_code, key_path.rstrip()
def get_ssh_port_for_vm(vm_name, path): """ Gets ssh port for vm Args: vm_name path to Vagrantfile Returns: return_code, command output Example Usage: my_class_var.get_ssh_port_for_vm(vm_name, path) """ cmd = "vagrant ssh-config --host %s | grep Port | awk '{print $2}'" % ( vm_name) return_code, port_no = service_utils.run_this(cmd, path) return return_code, port_no.rstrip()
def uninstall_gem(gem): """ Uninstall a specific gem. Args: gem(str): gem to uninstall Returns: returncode (int) -- 0 if successful, failure otherwise myinfo (str) -- stderr/stdout logs of the attempted gem uninstall Example Usage: >>> print uninstall_gem("bundler") 0 """ slab_logger.log(15, 'Uninstalling gem %s' % gem) returncode, myinfo = service_utils.run_this("gem uninstall -aIx %s" % (gem)) return returncode
def check_devenv(): """ Check that there is a proper development environment installed. Returns: True -- if a Redhat Variant or Windows False -- Ubuntu or other machines Example Usage: >>> print check_devenv() True """ slab_logger.log(15, 'Determining OS environment') if os.name == "posix": # this only works for RedHat and its variants. It does not work for Ubuntu. returncode, cmd_info = service_utils.run_this("yum list ruby-devel") if returncode == 0: return True return False return True
def create_project(self): """ Create the project on the gerrit server. Returns: ( int : return code str : output string if any. ) """ slab_logger.log(15, 'Creating project %s on gerrit' % self.get_reponame()) hostname = self.gsrvr['hostname'] port = self.gsrvr['port'] cmd = "ssh -p {} {}@{} gerrit create-project {}".format( port, self.username, hostname, self.get_reponame()) ret_code, ret_str = service_utils.run_this(cmd) return (ret_code, ret_str)
def code_review(self, number): """ Code review the given gerrit number Args: number -- Gerrit code review number Raises: GerritFnException -- If Unable to find the gerrit review number. """ slab_logger.debug('Pulling gerrit review %i for code review' % number) project = filters.OrFilter() project.add_items('project', [self.prjname]) other = filters.Items() other.add_items('change', number) key = self.getkey() query = reviews.Query(self.hostname, self.port, self.user, key) for review in query.filter(project, other): if 'type' in review.keys() and review['type'] == 'error': raise GerritFnException(review['message']) ref = review["currentPatchSet"]["ref"] tdir = "/tmp/{}".format(self.prjname) user = self.user host = self.hostname + ":" + str(self.port) # now do a git clone without checkout cmd = "rm -rf {};".format(tdir) pkey = "GIT_SSH_COMMAND=\"ssh -i {}\"".format(key) cmd += "{} git clone --no-checkout ssh://{}@{}/{} {};".format( pkey, self.user, host, self.prjname, tdir) cmd += "cd {};".format(tdir) cmd += "{} git fetch ssh://{}@{}/{} {}".format( pkey, user, host, self.prjname, ref) cmd += " && {} git format-patch -1 --stdout FETCH_HEAD".format( pkey) # now run the command and get output ret_code, ret_str = service_utils.run_this(cmd) if not ret_code: slab_logger.log(25, ret_str) else: raise GerritFnException(ret_str)
def setup_gems(path, ccsdata_repo=0): """ Set up gems in Gemfile to be used by bundler. Runs `bundle install` to setup gems in ccs-data if repo is present, and in home directory otherwise. Args: path (str): The path to your working .stack directory. Typically, this looks like ./servicelab/servicelab/.stack where "." is the path to the root of the servicelab repository. ccsdata_repo (int): set to 0 if ccs-data repo exists, 1 if it doesn't Returns: returncode (int): 0 -- Success 1 -- Failure, caused by failed bundle install Example Usage: >>> print setup_gems("/Users/aaltman/Git/servicelab/servicelab/.stack", 0) 0 #bundle install will have been performed in the servicelab root directory """ slab_logger.log(15, 'Setting up gems in Gemfile to be used by bundler') check_for_gems("bundler") if ccsdata_repo == 0: path_to_reporoot = os.path.join(path, "services", "ccs-data") else: path_to_reporoot = os.path.split(path) path_to_reporoot = os.path.split(path_to_reporoot[0]) path_to_reporoot = path_to_reporoot[0] returncode, myinfo = service_utils.run_this("bundle install", path_to_reporoot) if returncode == 1: slab_logger.error("Error on bundle install: %s" % (myinfo)) return (returncode) else: return (returncode)
def public_key(fname): """Public Key Creation via SSL. Args: fname (str): Certificate file ssl uses to create public key. Returns: returncode (int): 0 -- Success 1 -- Failure, can't create key (possibly invalid certificate) cmd_info (str): stdout/stderr log of pubkey-creation command Example Usage: >>> print publickey(public_key_pem) (0, "") """ slab_logger.loc(15, 'Creating ssl public key') cmd = "openssl x509 -inform pem -in %s -pubkey -noout" % (fname) cmd_returncode, cmd_info = service_utils.run_this(cmd) if cmd_returncode > 0: slab_logger.error(cmd_info) return (cmd_returncode, cmd_info)
def decrypt(pub_fname, priv_fname, data): """Decrypts data using the private key. Args: pub_fname (str): Public Key priv_fname (str): Private Key data (str): data that will be encrypted by the public key Returns: returncode (int): 0 -- Success 1 -- Failure, can't create key (possibly invalid certificate) cmd_info (str): stdout/stderr log of pubkey-creation command Example Usage: >>> print decrypt(private_key_pem) (0, "") #outputs decrypted data """ slab_logger.log(15, 'Decrptying data with ssl private key') code = "\"" code = code + "require 'openssl'; require 'base64';" code = code + "public_key_pem = File.read '" + pub_fname + "';" code = code + "public_key_x509 = OpenSSL::X509::Certificate.new(public_key_pem);" code = code + "private_key_pem = File.read '" + priv_fname + "';" code = code + "private_key_rsa = OpenSSL::PKey::RSA.new(private_key_pem);" code = code + "enc_value = Base64.decode64('" + data + "');" code = code + "pkcs7 = OpenSSL::PKCS7.new(OpenSSL::ASN1::decode(enc_value));" code = code + "print pkcs7.decrypt(private_key_rsa, public_key_x509);" code = code + "\"" cmd = "ruby -e " + code cmd_returncode, cmd_info = service_utils.run_this(cmd) if cmd_returncode > 0: slab_logger.error(cmd_info) return (cmd_returncode, cmd_info)