def check_for_gems(gem):
""" Check that a gem exists.
Before checking for gem, runs `type gem` command to check that gem commands are
available.
Outputs log as it checks for gem.
Args:
gem(str): gem to search for
Returns:
returncode (int) -- 0 if successful find, 1 otherwise (either not found or gem
command unavailable)
Example Usage:
>>> print check_for_gems("bundler")
0
"""
slab_logger.log(15, 'Checking for gem %s' % gem)
returncode, myinfo = service_utils.run_this("type gem")
if returncode == 0:
returncode_b, myinfo_b = service_utils.run_this(
'gem list | grep -q %s' % (gem))
if returncode_b == 1:
slab_logger.error("Cannot find gem(%s): %s" % (gem, myinfo_b))
return (returncode_b)
else:
return (returncode_b)
else:
slab_logger.error("Cannot find gem command: %s" % (myinfo))
return (returncode)
def setup_ruby(username=None):
""" Ruby set up.
Installs rvm
Sets username if none set.
Args:
username (str): defaults to None
Returns:
Returns no variables.
Example Usage:
>>> print setup_ruby()
#rvm installed
#username is set to aaltman
"""
slab_logger.log(15, 'Installing rvm')
# Note: I needed this on my CentOS7 machine, but not my MAC.
# Being allowed to fail for now.
service_utils.run_this("gpg2 --keyserver hkp://keys.gnupg.net \
--recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3")
returncode = service_utils.run_this("curl -L get.rvm.io | bash -s stable")
if returncode > 0:
slab_logger.error("Failed to install RVM.")
return (1)
else:
if username:
pass
else:
returncode, username = helper_utils.get_gitusername(ctx.path)
if returncode > 0:
slab_logger.error("Couldn't set user.")
def __init__(
self,
vm_name,
path,
provider="virtualbox",
default_vbox="Cisco/rhel-7",
default_vbox_url='http://cis-kickstart.cisco.com/ccs-rhel-7.box'):
"""
Constructor for Vagrant Class
Assigns member variables to constructor parameters.
Sets up Vagrant Client.
Args:
self -- self-referencing pointer
vname -- name of the virtual machine
path -- The path to your working .stack directory. Typically,
this looks like ./servicelab/servicelab/.stack where "."
is the path to the root of the servicelab repository.
provider -- vagrant provider, defaults to virtualbox
default_vbox -- default virtualbox, defaults to "Cisco/Rhel7"
default_vbox_url -- hyperlink to the default_vbox
"""
self.vm_name = vm_name
self.provider = provider
self.default_vbox = default_vbox
self.default_vbox_url = default_vbox_url
self.path = path
# Note: The quiet is so we know what's happening during
# vagrant commands in the term.
# Note: Setup vagrant client.
vagrant_dir = path
self.v = vagrant.Vagrant(root=vagrant_dir,
quiet_stdout=False,
quiet_stderr=False)
# check for installed plugins
# over here we will install the plugins.
# this is an extra step but by this we can avoid extra command repitition
# the other alternative is to put it in vagrantfile but this can be a problem
# as we may have to exec over the running process
# please see stack discussion
# http://stackoverflow.com/questions/19492738/demand-a-vagrant-plugin-within-the-vagrantfile
req_lst = ["vagrant-hostmanager", "vagrant-openstack-provider"]
plugin_lst = self.v.plugin_list()
for plugin in req_lst:
if plugin not in plugin_lst:
service_utils.run_this(
'vagrant plugin install {}'.format(plugin))
def create_repo(self):
"""
create a repo on the local machine. Returns -1 if the repo creation fails.
Returns:
(
int : return code
str : output string if any.
)
"""
slab_logger.log(15, 'Creating local repo %s' % self.get_reponame())
hostname = self.gsrvr['hostname']
port = self.gsrvr['port']
# please see https://code.google.com/p/gerrit/issues/detail?id=1013
# for -no-checkout option.
cmd = "git clone --no-checkout --depth=1 "
cmd += "ssh://{}@{}:{}/{} {}/.tmp".format(self.username, hostname,
port, self.get_reponame(),
self.get_reponame())
ret_code, ret_str = service_utils.run_this(cmd)
assert ret_code == 0, "unable to clone the project:" + ret_str
os.rename(os.path.join(self.get_reponame(), ".tmp", ".git"),
os.path.join(self.get_reponame(), ".git"))
shutil.rmtree(os.path.join(self.get_reponame(), ".tmp"))
return ret_code
def encrypt(pub_fname, data):
"""Encrypts data using the public key.
Args:
pub_fname (str): Public Key
data (str): data that will be encrypted by the public key
Returns:
returncode (int):
0 -- Success
1 -- Failure, can't create key (possibly invalid certificate)
cmd_info (str): stdout/stderr log of pubkey-creation command
Example Usage:
>>> print encrypt(public_key_pem)
(0, "")
#Data encryption is also outputted
"""
slab_logger.log(15, 'Encrpyting data using ssl public key')
code = "\""
code = code + "require 'openssl'; require 'base64';"
code = code + "public_key_pem = File.read '" + pub_fname + "';"
code = code + "public_key = OpenSSL::X509::Certificate.new(public_key_pem);"
code = code + "cipher = OpenSSL::Cipher::AES.new(256, :CBC);"
code = code + "enc = OpenSSL::PKCS7::encrypt([public_key], '"
code = code + data + "', " + "cipher, OpenSSL::PKCS7::BINARY).to_der;"
code = code + "print Base64.encode64(enc).gsub(/\n/, '');"
code = code + "\""
cmd = "ruby -e " + code
cmd_returncode, cmd_info = service_utils.run_this(cmd)
if cmd_returncode > 0:
slab_logger.error(cmd_info)
return (cmd_returncode, cmd_info)
def compile_man_page(path, user, password):
"""
Grabs data from confluence pages and servicelab docs and leverages sphinx to
convert them into a single man page that will be queried for high level info.
"""
slab_logger.log(15, 'Building data for man pages')
path_to_docs = os.path.split(path)[0]
path_to_docs = os.path.split(path_to_docs)[0]
path_to_docs = os.path.join(path_to_docs, 'docs')
man_yaml = _load_slabmanyaml(path)
# The man_contents.rst.inc file follows Sphinx syntax for enumerating the
# files that will be included in the final man page
with open(os.path.join(path_to_docs, "man_contents.rst.inc"), "w") as manf:
manf.write(".. toctree::\n :maxdepth: 2\n\n")
manf.writelines(" " + elem + "\n" for elem in man_yaml["slab_docs"])
# Manually download all desired sites, and leverage Sphinx's "make man" command
# to build a single man page using pages indexed above
for item in man_yaml['confluence_pages']:
url = 'https://confluence.sco.cisco.com/display/' + item
site_title = item.translate(None, '+')
file_title = item.translate(maketrans("+", "-")).lower()[4:]
content = requests.get(url, auth=(user, password))
if content.status_code != 200:
slab_logger.error(
"Unable to login to %s as user %s with supplied password." %
(url, user))
return
with open(
os.path.join(path_to_docs, 'man_pages', '%s.rst' % file_title),
'w') as manf:
manf.write(_parse_confluence_website(content.text, site_title))
service_utils.run_this(
"echo ' man_pages/%s' >> man_contents.rst.inc" % file_title,
path_to_docs)
cmd = "sed -i -e \"s/master_doc = 'index'/master_doc = 'man_index'/g\" conf.py; " \
"make man;" \
"sed -i -e \"s/master_doc = 'man_index'/master_doc = 'index'/g\" conf.py;"
service_utils.run_this(cmd, path_to_docs)
def download_template(self):
"""
Download the created project from the gerrit server
"""
hostname = self.gsrvr['hostname']
port = self.gsrvr['port']
cmd = "git clone --depth=1 "
cmd += "ssh://{}@{}:{}/{}".format(self.username, hostname, port,
self.get_reponame())
ret_code, ret_str = service_utils.run_this(cmd)
assert ret_code == 0, "unable to get project template project:" + ret_str
def repo_list(self):
""" Generate list of all repos on gerrit server."""
slab_logger.debug('Generating list of all repos on gerrit')
key = self.getkey()
cmd = "ssh -i {} -p {} {}@{} gerrit ls-projects".format(
key, self.port, self.user, self.hostname)
# now run the command and get output
ret_code, ret_str = service_utils.run_this(cmd)
if ret_code:
raise GerritFnException(ret_str)
return ret_str.split()
def download_template(self):
"""
Download the service-helloworld-ansible template from the gerrit server
"""
slab_logger.log(15,
'Downloading the service-helloworld-ansible template')
hostname = self.gsrvr['hostname']
port = self.gsrvr['port']
cmd = "git clone --depth=1 "
cmd += "ssh://{}@{}:{}/service-helloworld-puppet {}".format(
self.username, hostname, port, self.get_reponame())
ret_code, ret_str = service_utils.run_this(cmd)
assert ret_code == 0, "unable to get puppet template project:" + ret_str
def copy_file_to_vm(file_name, vm_name, file_path, port_no, vm_path, path,
key_path):
"""
Gets ssh port for vm
Args:
vm_name
path to Vagrantfile
Returns:
return_code, command output
Example Usage:
my_class_var.copy_file_to_vm(file_path, port_no, vm_path, path, key_path)
"""
cmd_string = "scp -i %s -P %s %s [email protected]:%s"
cmd = cmd_string % (key_path, port_no, file_path, "/tmp")
service_utils.run_this(cmd, path)
cmd_string = "vagrant ssh %s -c \"sudo cp /tmp/%s %s\""
cmd = cmd_string % (vm_name, file_name, vm_path)
return service_utils.run_this(cmd, path)
def get_ruby_version():
""" Returns the version of Ruby.
Returns:
Returns ruby version # or nothing if command `ruby -v` fails.
Example Usage:
>>> print get_ruby_version()
2.1.6
"""
slab_logger.log(15, 'Determining ruby version')
returncode, cmd_info = service_utils.run_this('ruby -v')
if returncode != 0:
return None
match = re.search("[0-9]+.[0-9]+.[0-9]+", cmd_info)
return match.group(0)
def _get_number_of_matches(section_name, path_to_man_page, query_str):
"""
Return the number of times query is found in the man page section
"""
slab_logger.log(15, 'Determining number of query matches')
# Equivalent to _view_section, except for the added grep command which
# ignores case and outputs matches, which are then counted by wc
section_name1 = section_name.replace(' ', r'\ ')
section_name2 = "^" + ".*".join(section_name) + "$"
cmd_view = "man -P 'less -p ^%s' %s | " % (section_name1, path_to_man_page)
cmd_view += "sed -n -e '/%s/,/^[^ \\t\\r\\n\\v\\f]/ p' | " % (
section_name2)
cmd_view += "sed -e '$d' | grep -io %s | wc -l " % (query_str)
# ignoring the error. just printing the output. this should be reviewed
_, output = service_utils.run_this(cmd_view)
return int(output.strip())
def get_ssh_key_path_for_vm(vm_name, path):
"""
Gets ssh key for vm
Args:
vm_name
path to Vagrantfile
Returns:
return_code, command output (key path)
Example Usage:
my_class_var.get_ssh_key_path_for_vm(vm_name, path)
"""
cmd = "vagrant ssh-config | grep %s | grep IdentityFile"\
" | awk '{print $2}'" % (vm_name)
return_code, key_path = service_utils.run_this(cmd, path)
return return_code, key_path.rstrip()
def get_ssh_port_for_vm(vm_name, path):
"""
Gets ssh port for vm
Args:
vm_name
path to Vagrantfile
Returns:
return_code, command output
Example Usage:
my_class_var.get_ssh_port_for_vm(vm_name, path)
"""
cmd = "vagrant ssh-config --host %s | grep Port | awk '{print $2}'" % (
vm_name)
return_code, port_no = service_utils.run_this(cmd, path)
return return_code, port_no.rstrip()
def uninstall_gem(gem):
""" Uninstall a specific gem.
Args:
gem(str): gem to uninstall
Returns:
returncode (int) -- 0 if successful, failure otherwise
myinfo (str) -- stderr/stdout logs of the attempted gem uninstall
Example Usage:
>>> print uninstall_gem("bundler")
0
"""
slab_logger.log(15, 'Uninstalling gem %s' % gem)
returncode, myinfo = service_utils.run_this("gem uninstall -aIx %s" %
(gem))
return returncode
def check_devenv():
""" Check that there is a proper development environment installed.
Returns:
True -- if a Redhat Variant or Windows
False -- Ubuntu or other machines
Example Usage:
>>> print check_devenv()
True
"""
slab_logger.log(15, 'Determining OS environment')
if os.name == "posix":
# this only works for RedHat and its variants. It does not work for Ubuntu.
returncode, cmd_info = service_utils.run_this("yum list ruby-devel")
if returncode == 0:
return True
return False
return True
def create_project(self):
"""
Create the project on the gerrit server.
Returns:
(
int : return code
str : output string if any.
)
"""
slab_logger.log(15,
'Creating project %s on gerrit' % self.get_reponame())
hostname = self.gsrvr['hostname']
port = self.gsrvr['port']
cmd = "ssh -p {} {}@{} gerrit create-project {}".format(
port, self.username, hostname, self.get_reponame())
ret_code, ret_str = service_utils.run_this(cmd)
return (ret_code, ret_str)
def code_review(self, number):
""" Code review the given gerrit number
Args:
number -- Gerrit code review number
Raises:
GerritFnException -- If Unable to find the gerrit review number.
"""
slab_logger.debug('Pulling gerrit review %i for code review' % number)
project = filters.OrFilter()
project.add_items('project', [self.prjname])
other = filters.Items()
other.add_items('change', number)
key = self.getkey()
query = reviews.Query(self.hostname, self.port, self.user, key)
for review in query.filter(project, other):
if 'type' in review.keys() and review['type'] == 'error':
raise GerritFnException(review['message'])
ref = review["currentPatchSet"]["ref"]
tdir = "/tmp/{}".format(self.prjname)
user = self.user
host = self.hostname + ":" + str(self.port)
# now do a git clone without checkout
cmd = "rm -rf {};".format(tdir)
pkey = "GIT_SSH_COMMAND=\"ssh -i {}\"".format(key)
cmd += "{} git clone --no-checkout ssh://{}@{}/{} {};".format(
pkey, self.user, host, self.prjname, tdir)
cmd += "cd {};".format(tdir)
cmd += "{} git fetch ssh://{}@{}/{} {}".format(
pkey, user, host, self.prjname, ref)
cmd += " && {} git format-patch -1 --stdout FETCH_HEAD".format(
pkey)
# now run the command and get output
ret_code, ret_str = service_utils.run_this(cmd)
if not ret_code:
slab_logger.log(25, ret_str)
else:
raise GerritFnException(ret_str)
def setup_gems(path, ccsdata_repo=0):
""" Set up gems in Gemfile to be used by bundler.
Runs `bundle install` to setup gems in ccs-data if repo is present,
and in home directory otherwise.
Args:
path (str): The path to your working .stack directory. Typically,
this looks like ./servicelab/servicelab/.stack where "."
is the path to the root of the servicelab repository.
ccsdata_repo (int): set to 0 if ccs-data repo exists, 1 if it doesn't
Returns:
returncode (int):
0 -- Success
1 -- Failure, caused by failed bundle install
Example Usage:
>>> print setup_gems("/Users/aaltman/Git/servicelab/servicelab/.stack", 0)
0
#bundle install will have been performed in the servicelab root directory
"""
slab_logger.log(15, 'Setting up gems in Gemfile to be used by bundler')
check_for_gems("bundler")
if ccsdata_repo == 0:
path_to_reporoot = os.path.join(path, "services", "ccs-data")
else:
path_to_reporoot = os.path.split(path)
path_to_reporoot = os.path.split(path_to_reporoot[0])
path_to_reporoot = path_to_reporoot[0]
returncode, myinfo = service_utils.run_this("bundle install",
path_to_reporoot)
if returncode == 1:
slab_logger.error("Error on bundle install: %s" % (myinfo))
return (returncode)
else:
return (returncode)
def public_key(fname):
"""Public Key Creation via SSL.
Args:
fname (str): Certificate file ssl uses to create public key.
Returns:
returncode (int):
0 -- Success
1 -- Failure, can't create key (possibly invalid certificate)
cmd_info (str): stdout/stderr log of pubkey-creation command
Example Usage:
>>> print publickey(public_key_pem)
(0, "")
"""
slab_logger.loc(15, 'Creating ssl public key')
cmd = "openssl x509 -inform pem -in %s -pubkey -noout" % (fname)
cmd_returncode, cmd_info = service_utils.run_this(cmd)
if cmd_returncode > 0:
slab_logger.error(cmd_info)
return (cmd_returncode, cmd_info)
def decrypt(pub_fname, priv_fname, data):
"""Decrypts data using the private key.
Args:
pub_fname (str): Public Key
priv_fname (str): Private Key
data (str): data that will be encrypted by the public key
Returns:
returncode (int):
0 -- Success
1 -- Failure, can't create key (possibly invalid certificate)
cmd_info (str): stdout/stderr log of pubkey-creation command
Example Usage:
>>> print decrypt(private_key_pem)
(0, "")
#outputs decrypted data
"""
slab_logger.log(15, 'Decrptying data with ssl private key')
code = "\""
code = code + "require 'openssl'; require 'base64';"
code = code + "public_key_pem = File.read '" + pub_fname + "';"
code = code + "public_key_x509 = OpenSSL::X509::Certificate.new(public_key_pem);"
code = code + "private_key_pem = File.read '" + priv_fname + "';"
code = code + "private_key_rsa = OpenSSL::PKey::RSA.new(private_key_pem);"
code = code + "enc_value = Base64.decode64('" + data + "');"
code = code + "pkcs7 = OpenSSL::PKCS7.new(OpenSSL::ASN1::decode(enc_value));"
code = code + "print pkcs7.decrypt(private_key_rsa, public_key_x509);"
code = code + "\""
cmd = "ruby -e " + code
cmd_returncode, cmd_info = service_utils.run_this(cmd)
if cmd_returncode > 0:
slab_logger.error(cmd_info)
return (cmd_returncode, cmd_info)