def server_diff(hostA, hostB, source_file, remote_path):
global OverthereHostSession
global OperatingSystemFamily
global LocalConnectionOptions
global ObertherHostSession
global StringUtils
global Diff
#context.logOutput(remote_path)
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
local_session = OverthereHostSession(OverthereHost(localOpts))
sessionA = OverthereHostSession(hostA)
sessionB = OverthereHostSession(hostB)
try:
d1 = sessionA.remote_file(remote_path)
local_d1 = local_session.work_dir_file("hostA")
local_session.copy_to(d1, local_d1)
myFile = "%s" % (local_d1)
local_f1 = myFile.split(':')[1]
d2 = sessionB.remote_file(remote_path)
local_d2 = local_session.work_dir_file("hostB")
local_session.copy_to(d2, local_d2)
myFile = "%s" % (local_d2)
local_f2 = myFile.split(':')[1]
response = local_session.execute("diff %s %s" % (local_f1, local_f2), check_success=False)
finally:
sessionA.close_conn()
sessionB.close_conn()
local_session.close_conn()
# End try
return response.stdout
def docker_machine_inspect(machine_name):
print "Inspect docker '{0}' machine ".format(machine_name)
command_line = "docker-machine inspect {0}".format(machine_name)
local_opts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(local_opts)
session = OverthereHostSession(host)
print "Executing '{0}'....".format(command_line)
try:
response = session.execute(command_line, check_success=False)
out = ""
for s in response.stdout:
out = "%s %s\n" % (out, s)
print out
for s in response.stderr:
print "ERR", s
if response.rc != 0:
sys.exit(response.rc)
return json.loads(out)
except:
tb = traceback.format_exc()
print "Error"
print tb
finally:
session.close_conn()
def server_diff(hostA, hostB, source_file, remote_path):
global OverthereHostSession
global OperatingSystemFamily
global LocalConnectionOptions
global ObertherHostSession
global StringUtils
global Diff
#context.logOutput(remote_path)
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
local_session = OverthereHostSession(OverthereHost(localOpts))
sessionA = OverthereHostSession(hostA)
sessionB = OverthereHostSession(hostB)
try:
d1 = sessionA.remote_file(remote_path)
local_d1 = local_session.work_dir_file("hostA")
local_session.copy_to(d1, local_d1)
myFile = "%s" % (local_d1)
local_f1 = myFile.split(':')[1]
d2 = sessionB.remote_file(remote_path)
local_d2 = local_session.work_dir_file("hostB")
local_session.copy_to(d2, local_d2)
myFile = "%s" % (local_d2)
local_f2 = myFile.split(':')[1]
response = local_session.execute("diff %s %s" % (local_f1, local_f2),
check_success=False)
finally:
sessionA.close_conn()
sessionB.close_conn()
local_session.close_conn()
# End try
return response.stdout
def get_associated_pods(ci):
pods = []
session = OverthereHostSession(target_host)
command_line = "{0} get {1} -l=component={2} -o json".format(get_kubectl_command(ci.container), 'pods', ci.name)
print command_line
response = session.execute(command_line)
if response.rc == 0:
data = json.loads(" ".join(response.stdout))
for item in data['items']:
pods.append(item['metadata']['name'])
return pods
def get_pod_events(ci, pod_name):
events = []
session = OverthereHostSession(target_host)
command_line = "{0} get {1} --field-selector involvedObject.name={2} -o json".format(
get_kubectl_command(ci.container), 'event',
pod_name)
response = session.execute(command_line)
if response.rc == 0:
data = json.loads(" ".join(response.stdout))
for item in data['items']:
events.append("{type} {message}".format(**item))
return events
def execute_script(server):
print "Running script [%s] on server instance [%s]" % (scriptToExecute, server.getProperty("serverName"))
sshOpts = SshConnectionOptions(server.getProperty("address"), server.getProperty("username"), password=server.getProperty("password"))
host = OverthereHost(sshOpts)
session = OverthereHostSession(host)
response = session.execute([scriptToExecute], check_success=False)
if response.rc != 0:
print "Failed to execute command"
print response.stderr
else:
print "Response", str(response.stdout)
session.close_conn()
class XLRunner:
def __init__(self, xlpath, server):
self.xlpath = xlpath
self.server = server
self.session = None
def get_xl_command(self):
options = '--xl-deploy-url {0} --xl-deploy-username {1} --xl-deploy-password {2}'.format(
self.server['url'], self.server['username'],
self.server['password'])
xl = "{0}/xl {1}".format(self.xlpath, options)
return xl
def parameters(self, content, values):
uploaded_file = self.session.upload_text_content_to_work_dir(
content, "content.yaml")
parameters = " -f " + uploaded_file.getPath()
if len(values) > 0:
parameters = parameters + " --values " + ",".join(
["{0}={1}".format(k, v) for k, v in values.items()])
return parameters
def command_line(self, content, values):
return "{0} apply {1}".format(self.get_xl_command(),
self.parameters(content, values))
def apply(self, content, values):
try:
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(localOpts)
self.session = OverthereHostSession(host,
stream_command_output=False)
command_line = self.command_line(content, values)
#print command_line
uploaded_runner = self.session.upload_text_content_to_work_dir(
command_line, 'xldeploy_xl_runner.sh', executable=True)
#print uploaded_runner.path
response = self.session.execute(command_line, check_success=False)
print "\n```"
print "\n ".join(response.stdout)
print "\n ".join(response.stderr)
print "```\n "
rc = response.rc
if response.rc > 0:
sys.exit(rc)
finally:
if not self.session == None:
self.session.close_conn()
def execute(self,deployed):
try:
session = OverthereHostSession(self.helmclient.host,stream_command_output=False)
command_line = self.command_line(session,deployed)
print command_line
uploaded_runner = session.upload_text_content_to_work_dir(command_line,'xldeploy_helm.sh',executable=True)
print uploaded_runner.path
response = session.execute(command_line,check_success=False)
print "\n ".join(response.stdout)
print "\n ".join(response.stderr)
rc = response.rc
if response.rc > 0:
sys.exit(rc)
finally:
session.close_conn()
def gke_describe(name, target):
command_line = "gcloud container clusters describe {0} --format=json".format(
name)
print command_line
host = target.container
session = OverthereHostSession(host)
print "Executing '{0}'....".format(command_line)
try:
response = session.execute(command_line)
return json.loads(" ".join(response.stdout))
except:
tb = traceback.format_exc()
print "Error"
print tb
finally:
session.close_conn()
def docker_machine_env(machine_name):
print "Env docker '{0}' machine ".format(machine_name)
command_line = "docker-machine env {0}".format(machine_name)
local_opts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(local_opts)
session = OverthereHostSession(host)
print "Executing '{0}'....".format(command_line)
try:
response = session.execute(command_line)
return to_map(response.stdout)
except:
tb = traceback.format_exc()
print "Error"
print tb
finally:
session.close_conn()
def execute_docker_compose(context,
composed,
application,
command,
options=''):
session = OverthereHostSession(composed.container.host,
stream_command_output=True,
execution_context=context)
runtime = {
'application':
application,
'command':
command,
'options':
options,
'uploaded_compose_file':
session.upload_file_to_work_dir(composed.file),
'dockerHost':
composed.container.dockerHost,
'cert_pem':
session.upload_text_content_to_work_dir(composed.container.certPem,
"cert.pem"),
'ca_pem':
session.upload_text_content_to_work_dir(composed.container.caPem,
"ca.pem"),
'key_pem':
session.upload_text_content_to_work_dir(composed.container.keyPem,
"key.pem")
}
print "docker-compose {0}....".format(runtime['command'])
command_line = "docker-compose --file {r[uploaded_compose_file].path} --project-name {r[application]} --tlscacert {r[ca_pem].path} --tlscert {r[cert_pem].path} --tlskey {r[key_pem].path} --tlsverify -H {r[dockerHost]} --skip-hostname-check {r[command]} {r[options]} ".format(
r=runtime)
print '---------------------------'
print command_line
print '---------------------------'
response = session.execute(command_line)
if response.rc != 0:
sys.exit(response.rc)
def run(self, verb, parameters, json_output=False, raise_on_fail=False):
session = OverthereHostSession(self.cluster.kubectlHost)
command_line = "{0} {1} {2}".format(self.get_kubectl_command(), verb,
parameters)
if json_output:
command_line = command_line + " -o json"
print command_line
response = session.execute(command_line,
suppress_streaming_output=True,
check_success=False)
if response.rc == 0:
stdout = "\n".join(response.stdout)
if json_output:
return json.loads(stdout)
else:
return stdout
else:
if raise_on_fail:
raise Exception("Kubectl Error when running '{0}':{1}".format(
command_line, '\n'.join(response.stderr)))
else:
return '\n'.join(response.stderr)
def execute(self, xmlaccessscript, config_uri=None):
session = OverthereHostSession(self.host,
stream_command_output=False,
execution_context=self.exec_context)
with session:
request_file = session.upload_text_content_to_work_dir(
xmlaccessscript, "request.xml")
response_file = session.work_dir_file("response.xml")
fs = session.os.fileSeparator
executable = "%s%sbin%sxmlaccess%s" % (self.wp_home, fs, fs,
session.os.scriptExtension)
wp_url = self.resolve_config_url(config_uri)
cmd_line = [
executable, "-user", self.wp_user, "-password",
self.wp_password, "-url", wp_url
]
cmd_line.extend(
["-in", request_file.path, "-out", response_file.path])
resp = session.execute(cmd_line, check_success=False)
response_xml = ""
if response_file.exists():
response_xml = session.read_file(response_file.path)
if resp.rc != 0:
self.exec_context.logError("XML Access failed!!!")
self.log_with_header("XML Access response:", response_xml,
True)
self.log_with_header("Standard out",
StringUtils.concat(resp.stdout))
self.log_with_header("Standard error",
StringUtils.concat(resp.stderr))
cmd_line[4] = '******' # Password is 5th element in array
self.log_with_header("Executed command line",
StringUtils.concat(cmd_line, " "))
sys.exit(1)
return response_xml
def execute(self, xmlaccessscript, config_uri=None):
session = OverthereHostSession(self.host, stream_command_output=False, execution_context=self.exec_context)
with session:
request_file = session.upload_text_content_to_work_dir(xmlaccessscript, "request.xml")
response_file = session.work_dir_file("response.xml")
fs = session.os.fileSeparator
executable = "%s%sbin%sxmlaccess%s" % (self.wp_home, fs, fs, session.os.scriptExtension)
wp_url = self.resolve_config_url(config_uri)
cmd_line = [executable, "-user", self.wp_user, "-password", self.wp_password, "-url", wp_url]
cmd_line.extend(["-in", request_file.path, "-out", response_file.path])
resp = session.execute(cmd_line, check_success=False)
response_xml = ""
if response_file.exists():
response_xml = session.read_file(response_file.path)
if resp.rc != 0:
self.exec_context.logError("XML Access failed!!!")
self.log_with_header("XML Access response:", response_xml, True)
self.log_with_header("Standard out", StringUtils.concat(resp.stdout))
self.log_with_header("Standard error", StringUtils.concat(resp.stderr))
cmd_line[4] = '******' # Password is 5th element in array
self.log_with_header("Executed command line", StringUtils.concat(cmd_line, " "))
sys.exit(1)
return response_xml
class UnixBuildServer(object):
def __init__(self, server):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.base_working_directory = server['BaseWorkingDirectory']
self.working_directory = self.base_working_directory
self.create_working_directory(self.base_working_directory)
workDir = self.session.remote_file(self.base_working_directory)
self.session.get_conn().setWorkingDirectory(workDir)
@staticmethod
def createServer(server):
return UnixBuildServer(server)
def set_working_directory(self, relative_directory_path):
self.working_directory = "%s/%s" % (self.base_working_directory, relative_directory_path)
self.create_working_directory(self.working_directory)
workDir = self.session.remote_file(self.working_directory)
self.session.get_conn().setWorkingDirectory(workDir)
# main interface
# session related methods
def close_session(selfs):
self.session.close()
def create_working_directory(self, dir_full_path):
response = self.session.execute("/bin/mkdir -p %s" % (dir_full_path))
# File Handeling
def read_file_in_workspace(self, file_name):
print "not yet implemented"
def write_file_in_workspace(self, file_name, content):
print "not yet implemented"
def create_directory(self, directory_name):
self.execute_command(["/bin/mkdir -p %s" % (directory_name)])
# command Execution
def filename_generator(self, size=9, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def get_tmp_script_filename(self):
return "%s.sh" % self.filename_generator()
def execute_command(self, command, environment_variables=None):
# get a BashScriptBuilder object
command_object = BashScriptBuilder()
# copy in the environment variables
if environment_variables is not None:
for key, value in environment_variables.items():
command_object.set_env_var(key, value)
# add the rest of the possible multiline command
command_object.add_lines(command)
executable_command = command_object.build()
print "executing command: %s " % (executable_command)
tmp_script_filename = self.get_tmp_script_filename()
self.session.upload_text_content_to_work_dir(command_object.build(), tmp_script_filename, executable=True)
#
response = self.session.execute("%s/%s" % (self.working_directory, tmp_script_filename), check_success=False, suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print response.stdout
print "unable to execute command %s" % (command)
raise Exception('unable to execute command')
else:
print "Response:", str(response.stdout)
return response
#
# THIS CODE AND INFORMATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS
# FOR A PARTICULAR PURPOSE. THIS CODE AND INFORMATION ARE NOT SUPPORTED BY XEBIALABS.
#
from __future__ import with_statement
from overtherepy import OverthereHostSession
from liquibase import *
session = OverthereHostSession(container.host, stream_command_output=True, execution_context=context)
with session:
cmd_line = build_cmd_line(container)
cmd_line.extend(['tag', tag])
print_cmd_line(cmd_line, context)
session.execute(cmd_line)
from com.xebialabs.overthere import OperatingSystemFamily
from overtherepy import LocalConnectionOptions, OverthereHost, OverthereHostSession
machine_name = thisCi.machineName
if machine_name is None:
raise ValueError("the 'Machine Name' property is empty")
print "Start docker '{0}' machine ".format(machine_name)
command_line = "docker-machine start {0}".format(machine_name)
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(localOpts)
session = OverthereHostSession(host)
print "Executing '{0}'....".format(command_line)
try:
response = session.execute(command_line, check_success=False)
for s in response.stdout:
print s
for s in response.stderr:
print s
except:
tb = traceback.format_exc()
print "Error"
print tb
finally:
session.close_conn()
if response.rc != 0:
sys.exit(response.rc)
class DarBuildServer(object):
def __init__(self, server):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.zipBinary = server['pathToZipExecutable']
self.workingDirectory = server['workingDirectory']
self.create_directory(self.workingDirectory)
workDir = self.session.remote_file(self.workingDirectory)
self.session.get_conn().setWorkingDirectory(workDir)
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server):
return DarBuildServer(server)
#
def init_dar(self, appName, appVersion):
workspace_root = self.create_dar_directory(appName, appVersion)
manifest_filename = "%s/deployit-manifest.xml" % workspace_root
self.write_dar_manifest(appName, appVersion, workspace_root)
def import_ear(self, appName, appVersion, deployable, url):
self.download_file_into_dar(appName, appVersion,deployable, str(url))
def create_dar_directory(self, appName, appVersion):
dirName = "%s/%s" % (appVersion, appName)
# check if the directory exists .. if it does we should go do something else.
# might want to do a better locking mechanism
if self.dir_exists(dirName):
print "unable to create dar directory: %s/%s/%s" % (self.workingDirectory, appVersion, appName)
raise Exception('unable to create Dar Package Directory')
else:
self.create_directory(dirName)
return dirName
def download_file_into_dar(self, appName, appVersion, deployable, url):
#filename = url.split('/')[-1]
filename = os.path.basename(url)
outfile = "%s/%s/%s/%s" % (appVersion, appName,deployable, filename)
dirName = "%s/%s/%s" % (appVersion, appName,deployable)
if self.dir_exists(dirName):
print "output dir already exists: %s" % (dirName)
else:
self.create_directory(dirName)
self.execute_command("/usr/bin/curl -L -o %s %s" % (outfile, url))
def read_manifest(self, appName, appVersion):
file_name = "%s/%s/%s/deployit-manifest.xml" % (self.workingDirectory, appVersion, appName)
return self.session.read_file(file_name, encoding="UTF-8")
def write_manifest(self, appName, appVersion, content):
file_name = "%s/%s/deployit-manifest.xml" % (appVersion, appName)
self.write_to_file(file_name, content)
def create_directory(self, dirName):
self.execute_command("/bin/mkdir -p %s" % (dirName))
def create_file(self, fileName, content=None):
if content:
self.write_to_file(fileName, str(content))
else:
self.execute_command("/bin/touch %s" % (fileName))
def write_to_file(self, fileName, content):
remoteFile = self.session.remote_file("%s/%s" % (self.workingDirectory, fileName))
self.session.copy_text_to_file(str(content), remoteFile)
def dir_exists(self, dirName):
print dirName
command = "[ -d %s ]" % (dirName)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc == 0 :
return True
else:
return False
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response:", str(response.stdout)
# self.switch_working_directory()
return response
def write_dar_manifest(self, appName, appVersion, workspace_root):
filename = "./%s/deployit-manifest.xml" % (workspace_root)
file_content = self.basic_dar_manifest_template().substitute(appVersion=appVersion, appName=appName)
self.create_file(filename, file_content)
def basic_dar_manifest_template(self):
xml_template = '<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n'
xml_template += ' <udm.DeploymentPackage version=\"$appVersion\" application=\"$appName\"> \n'
xml_template += ' <orchestrator /> \n'
xml_template += ' <deployables/> \n'
xml_template += ' </udm.DeploymentPackage> \n'
return Template(xml_template)
def package_dar(self, appName, appVersion):
command = "if [ -f %s_%s.dar ] \n"
command += " then rm -rf %s_%s.dar \n"
command += "fi \n"
command += "cd %s/%s \n" % (appVersion, appName)
command += "/usr/bin/zip -r %s/%s_%s.dar *" % (self.workingDirectory, appName, appVersion.replace('.','_'))
self.execute_multiline_command(command)
def write_exec_script(self, commands, script):
self.session.upload_text_content_to_work_dir(self, commands, script, executable=False)
def execute_multiline_command(self, command):
tmp_filename = self.filename_generator()
self.write_to_file(tmp_filename, command)
self.execute_command("chmod +x %s" % (tmp_filename))
self.execute_command("./%s" % (tmp_filename))
def filename_generator(self, size=9, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def upload_dar_package(self, appName, appVersion, xldeployServer):
dar_file_name = "%s_%s.dar" % (appName, appVersion.replace('.','_'))
command = "/usr/bin/curl -u %s:%s -X POST -H \"content-type:multipart/form-data\" %s/package/upload/%s -F fileData=@./%s " % (xldeployServer['username'], xldeployServer['password'], xldeployServer['url'], dar_file_name, dar_file_name)
print command
self.execute_command(command)
class TestOverthereSession(object):
# preparing to test
def setUp(self):
self._linuxhost = OverthereHost(
LocalConnectionOptions(os=OperatingSystemFamily.UNIX))
self._session = OverthereHostSession(self._linuxhost)
self._session.logger = OverthereSessionLogger(capture=True)
# ending the test
def tearDown(self):
self._session.close_conn()
def _clear_logs(self):
self._session.logger.output_lines = []
self._session.logger.error_lines = []
def assert_in_list(self, list, expect):
r = [s for s in list if str(s) == str(expect)]
eq_(len(r), 1, "expect [%s] to be in list [%s]" % (expect, str(list)))
def test_not_windows_host(self):
ok_(not self._session.is_windows())
def test_work_dir_cleanup(self):
workdir = self._session.work_dir()
ok_(workdir.exists())
workdir_path = workdir.path
self._session.close_conn()
ok_(not os.path.exists(workdir_path))
def test_read_write_file(self):
f = self._session.upload_text_content_to_work_dir(
"some text", "my.txt")
text = self._session.read_file(f.path)
eq_("some text", text)
def test_upload_executable_file(self):
f = self._session.upload_text_content_to_work_dir(
"#!/bin/sh\necho hello", "my.sh", executable=True)
r = self._session.execute(f.path)
self.assert_in_list(r.stdout, "hello")
def test_upload_classpath_resource(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/singleline.txt")
text = self._session.read_file(f.path)
eq_("some text 1", text)
def test_read_lines(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/multiline.txt")
lines = self._session.read_file_lines(f.path)
eq_(3, len(lines))
def test_execute(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
response = self._session.execute([f.path, "ping"])
eq_(response['rc'], 0)
eq_(response['stdout'][0], "Hi ping")
def test_execute_automatic_system_exit_on_failure(self):
success = False
try:
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
self._session.execute([f.path, "ping", "1"])
except SystemExit:
success = True
pass
eq_(success, True)
def test_execute_check_success_turned_off(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
response = self._session.execute("%s ping 1" % f.path,
check_success=False)
eq_(response.rc, 1)
eq_(response['stdout'][0], "Hi ping")
eq_(response['stdout'][1], "Exiting with 1")
def test_with_support(self):
s = OverthereHostSession(self._linuxhost)
with s:
work_dir = s.work_dir().path
eq_(os.path.exists(work_dir), True)
eq_(os.path.exists(work_dir), False)
def _local_file(self, resource):
url = Thread.currentThread().contextClassLoader.getResource(resource)
return self._session.local_file(File(url.toURI()))
def test_mkdirs_on_dir_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
eq_(os.path.exists(target.path + "/echo.sh"), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_file_copy(self):
target = self._session.work_dir_file("some/path/some.txt")
source = self._local_file("testfiles/echo.sh")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.parentFile.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_turned_off_on_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
success = False
try:
self._session.copy_to(source, target, mkdirs=False)
except:
success = True
eq_(success, True)
def test_delete_from_when_target_does_not_exit(self):
target = self._session.work_dir_file("some/path")
source = None
self._session.delete_from(source, target)
log_info = self._session.logger.output_lines
eq_(len(log_info), 1)
eq_(
log_info[0],
"Target [%s] does not exist. No deletion to be performed" %
target.path)
def test_delete_from_file_target(self):
target = self._session.work_dir_file("some/path/test.txt")
self._session.copy_text_to_file("xxx", target)
eq_(os.path.exists(target.path), True)
source = None
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
self.assert_in_list(log_info, "Deleting [%s]" % target.path)
def test_delete_from_folder_target(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(
log_info,
"Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting directory [%s]" % target.path)
def test_delete_from_with_external_resources(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
external_resource = self._session.work_dir_file("some/path/ext.txt")
self._session.copy_text_to_file("xxx", external_resource)
eq_(os.path.exists(external_resource.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(
log_info,
"Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(
log_info,
"Target directory [%s] is not shared, but still has content from an external source. Will not delete"
% target.path)
def test_copy_diff(self):
old = self._local_file("directorycompare/old")
new = self._local_file("directorycompare/new")
deployed_old = self._session.work_dir_file("olddeployed")
self._session.copy_to(old, deployed_old)
eq_(os.path.exists(deployed_old.path), True)
self._clear_logs()
diff = Diff.calculate_diff(old, new)
self._session.copy_diff(deployed_old.path, diff)
log_info = self._session.logger.output_lines
eq_(len(log_info), 17)
self.assert_in_list(log_info, "3 files to be removed.")
self.assert_in_list(log_info, "3 new files to be copied.")
self.assert_in_list(log_info, "2 modified files to be copied.")
self.assert_in_list(log_info, "Start removal of files...")
self.assert_in_list(
log_info, "Removing file %s/removefile.txt" % deployed_old.path)
self.assert_in_list(
log_info, "Removing directory %s/subdirremove" % deployed_old.path)
self.assert_in_list(
log_info,
"Removing file %s/subdirboth/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removal of files done.")
self.assert_in_list(log_info, "Start copying of new files...")
self.assert_in_list(
log_info, "Copying directory %s/subdirnew" % deployed_old.path)
self.assert_in_list(log_info,
"Copying file %s/newfile.txt" % deployed_old.path)
self.assert_in_list(
log_info,
"Copying file %s/subdirboth/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of new files done.")
self.assert_in_list(log_info, "Start copying of modified files...")
self.assert_in_list(
log_info, "Updating file %s/changedfile.txt" % deployed_old.path)
self.assert_in_list(
log_info,
"Updating file %s/subdirboth/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of modified files done.")
def test_execution_ctx_logging(self):
class ExecutionContext(object):
def __init__(self):
self.output_success = False
self.error_success = False
def logOutput(self, msg):
self.output_success = True
def logError(self, msg):
self.error_success = True
exec_ctx = ExecutionContext()
session = OverthereHostSession(self._linuxhost,
execution_context=exec_ctx)
session.logger.info("Check")
eq_(exec_ctx.output_success, True)
session.logger.error("Check")
eq_(exec_ctx.error_success, True)
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp"):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
self.RtcClient = server['pathToClientSoftware']
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server, dirname):
return ConfigRTCServer(server, dirname)
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" % (self.getWorkDirectory(), file_name))
return response
def execute_lscm_command(self, command, run_in_workdirectory=False):
print command
command = self.sub_placeholders(command)
print command
lscm_command="%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response", str(response.stdout)
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.execute_command("/bin/mkdir -p %s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" % (self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
print self.workDirectory
print self.getWorkDirectory()
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
values = {
'startDelay': smoketest.startDelay,
'maxRetries': smoketest.maxRetries,
'retryWaitInterval': smoketest.retryWaitInterval,
'target_url': context.getAttribute(deployed.id),
'expectedResponseText': smoketest.expectedResponseText
}
print values['target_url']
content = template.safe_substitute(values)
#print "-----------------"
#print content
#print "-----------------"
session = OverthereHostSession(target_host)
remote = session.upload_text_content_to_work_dir(content,
'benoit_aws_test.sh',
executable=True)
print remote.getPath()
response = session.execute(remote.getPath(), check_success=False)
for line in response.stdout:
print line
for line in response.stderr:
print line
rc = response.rc
if rc != 0:
print "Non zero Exit Code: {0}".format(rc)
sys.exit(rc)
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS
# FOR A PARTICULAR PURPOSE. THIS CODE AND INFORMATION ARE NOT SUPPORTED BY XEBIALABS.
#
from overtherepy import LocalConnectionOptions, OverthereHost, OverthereHostSession
from com.xebialabs.overthere import OperatingSystemFamily
import sys
import re
def to_map(stdout):
data = {}
for s in response.stdout:
if 'export' in s:
info = s.replace('export ', ' ').replace('"',
' ').strip().split('=')
data[str(info[0])] = str(info[1])
return data
session = OverthereHostSession(target.container.host)
response = session.execute("docker-machine env %s " % target.machineName)
data = to_map(response.stdout)
ip = re.findall(r'[0-9]+(?:\.[0-9]+){3}', data['DOCKER_HOST'])
if len(ip) == 1:
data['address'] = ip[0]
print "IP Address is %s " % data['address']
print data
session.close_conn()
context.setAttribute(context_key, data)
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS
# FOR A PARTICULAR PURPOSE. THIS CODE AND INFORMATION ARE NOT SUPPORTED BY XEBIALABS.
#
from overtherepy import LocalConnectionOptions, OverthereHost, OverthereHostSession
from com.xebialabs.overthere import OperatingSystemFamily
import sys
import re
def to_map(stdout):
data={}
for s in response.stdout:
if 'export' in s:
info = s.replace('export ',' ').replace('"',' ').strip().split('=')
data[str(info[0])]=str(info[1])
return data
session = OverthereHostSession(target.container.host)
response = session.execute("docker-machine env %s " % target.machineName)
data=to_map(response.stdout)
ip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', data['DOCKER_HOST'])
if len(ip) == 1:
data['address'] = ip[0]
print "IP Address is %s " % data['address']
print data
session.close_conn()
context.setAttribute(context_key, data)
cis = request.query['cis'].split(',')
logger.info(str(type(cis)))
logger.info(str(cis))
rtc_client = repositoryService.read(request.query["client"])
rtc_repo = repositoryService.read(request.query["repo"])
sshOpts = SshConnectionOptions(rtc_client.getProperty("host"),username = rtc_client.getProperty("username"), password = rtc_client.getProperty("password"))
host = OverthereHost(sshOpts)
session = OverthereHostSession(host)
workspace = RTCWorkspace.createRTCWorkspace(rtc_repo,rtc_client)
loc = "/var/tmp/"
#Step one: change to a new directory to place configs
#TODO: Decide whether script makes directory or directory is there ahead of time
stepone = session.execute("cd " + loc, check_success=True)
#Retrieve key value of exported DAR
keyval = request.query["ciid"]
addkey = "curl -o " + loc + keyval + ".dar " + "http://*****:*****@localhost:4516/deployit/internal/download/" + keyval
#Step three: Execute cURL command to download DAR based on key to new directory
stepthree = session.execute(addkey, check_success=True)
getxml = "unzip " + loc + keyval + ".dar -d " + loc + keyval
#Step four: Unzip the DAR
stepfour = session.execute(getxml, check_success=True)
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp"):
self.sshOpts = SshConnectionOptions(server['host'],
username=server['username'],
password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
self.RtcClient = server['pathToClientSoftware']
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server, dirname):
return ConfigRTCServer(server, dirname)
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" %
(self.getWorkDirectory(), file_name))
return response
def execute_lscm_command(self, command, run_in_workdirectory=False):
print command
command = self.sub_placeholders(command)
print command
lscm_command = "%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command,
check_success=False,
suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response", str(response.stdout)
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.execute_command(
"/bin/mkdir -p %s/%s/%s" %
(self.WorkDirBase, self.WorkDirTimeStamp, self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (
self.WorkDirBase, self.WorkDirTimeStamp, self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" %
(self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
print self.workDirectory
print self.getWorkDirectory()
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
return -1
attempts = 10
deployment_name = podname
if deployed.namespace is not None:
command_line = "{2} get deployment {0} --namespace={1} -o=json".format(
deployment_name, deployed.namespace, deployed.container.command)
else:
command_line = "{2} get deployment {0} -o=json".format(
deployment_name, deployed.namespace, deployed.container.command)
print command_line
try:
session = OverthereHostSession(deployed.container.host)
response = session.execute(command_line)
data = json.loads(" ".join(response.stdout))
condition = data['status']['conditions'][0]
print "Status {status} {reason}: {message}".format(**condition)
availableReplicas = get_available_replicas(data)
print "availableReplicas {0}/{1}".format(availableReplicas,
deployed.replicas)
if condition['status'] == "True":
print "Status Ok"
elif availableReplicas == deployed.replicas:
print "DONE replicas"
else:
inc_context()
cpt = get_value_context()
#
# Copyright 2021 XEBIALABS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
from overtherepy import OverthereHostSession
from java.io import File
import sys
session = OverthereHostSession(container.ansibleController.host)
local_session = OverthereHostSession(container.ansibleController.host)
source = session.local_file(File(container.privateKeyFile))
target = session.remote_file(target_path)
if target.exists():
print("File exists, delete it !")
target.delete()
print('Copy {0} -> {1}'.format(source, target))
session.copy_to(source, target)
chmod = "chmod 400 {0}".format(target_path)
print(chmod)
response = session.execute(chmod)
class DarBuildServer(object):
def __init__(self, server):
self.retryCount = server['retryCount']
if self.retryCount == 0:
self.retryCount = 1
self.retryBaseInterval = 60
self.sshOpts = SshConnectionOptions(server['host'],
username=server['username'],
privateKeyFile=server['key_file'],
password=server['password'],
port=server['port'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
tries = 0
while tries < self.retryCount:
tries += 1
try:
self.zipBinary = server['pathToZipExecutable']
self.workingDirectory = server['workingDirectory']
self.create_directory(self.workingDirectory)
workDir = self.session.remote_file(self.workingDirectory)
self.session.get_conn().setWorkingDirectory(workDir)
break
except:
if tries > 3:
print sys.exc_info()[0]
print sys.exc_info()[1]
if tries > self.retryCount:
print sys.exc_info()[2].format_stack()
print "we were unable to setup a connection to server: %s after %i retries" % (
server['host'], tries)
sys.exit(2)
sleeptime = self.retryBaseInterval * tries
print "retrying to setup a connection to server %s in %i seconds" % (
server['host'], sleeptime)
time.sleep(sleeptime)
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
def closeConnection(self):
print "closing ssh connection"
self.session.close_conn()
@staticmethod
def createServer(server):
return DarBuildServer(server)
#
def init_dar(self, appName, appVersion):
workspace_root = self.create_dar_directory(appName, appVersion)
manifest_filename = "%s/deployit-manifest.xml" % workspace_root
self.write_dar_manifest(appName, appVersion, workspace_root)
def delete_workspace(self, appName, appVersion):
dirName = "%s/%s" % (appVersion, appName)
response = self.execute_command("/bin/rm -rf %s" % (dirName))
response = self.execute_command("/bin/rm -rf %s" % (appVersion))
def import_ear(self, appName, appVersion, deployable, url):
self.download_file_into_dar(appName, appVersion, deployable, str(url))
def create_dar_directory(self, appName, appVersion):
dirName = "%s/%s" % (appVersion, appName)
# check if the directory exists .. if it does we should go do something else.
# might want to do a better locking mechanism
if self.dir_exists(dirName):
print "unable to create dar directory: %s/%s/%s" % (
self.workingDirectory, appVersion, appName)
raise Exception('unable to create Dar Package Directory')
else:
self.create_directory(dirName)
return dirName
def download_file_into_dar(self, appName, appVersion, deployable, url):
#filename = url.split('/')[-1]
filename = os.path.basename(url)
outfile = "%s/%s/%s/%s" % (appVersion, appName, deployable, filename)
dirName = "%s/%s/%s" % (appVersion, appName, deployable)
if self.dir_exists(dirName):
print "output dir already exists: %s" % (dirName)
else:
self.create_directory(dirName)
self.execute_command(
"/usr/bin/curl --retry 5 --retry-delay 2 -k -L -o %s %s" %
(outfile, url))
def read_manifest(self, appName, appVersion):
file_name = "%s/%s/%s/deployit-manifest.xml" % (self.workingDirectory,
appVersion, appName)
return self.session.read_file(file_name, encoding="UTF-8")
def write_manifest(self, appName, appVersion, content):
file_name = "%s/%s/deployit-manifest.xml" % (appVersion, appName)
self.write_to_file(file_name, content)
def create_directory(self, dirName):
self.execute_command("/bin/mkdir -p %s" % (dirName))
def create_file(self, fileName, content=None):
if content:
self.write_to_file(fileName, str(content))
else:
self.execute_command("/bin/touch %s" % (fileName))
def write_to_file(self, fileName, content):
remoteFile = self.session.remote_file(
"%s/%s" % (self.workingDirectory, fileName))
self.session.copy_text_to_file(str(content), remoteFile)
def dir_exists(self, dirName):
command = "[ -d %s ]" % (dirName)
response = self.session.execute(command,
check_success=False,
suppress_streaming_output=False)
if response.rc == 0:
return True
else:
return False
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command,
check_success=False,
suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print response.stdout
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response:", str(response.stdout)
print "Errors:", str(response.stderr)
# self.switch_working_directory()
return response
def write_dar_manifest(self, appName, appVersion, workspace_root):
filename = "./%s/deployit-manifest.xml" % (workspace_root)
file_content = self.basic_dar_manifest_template().substitute(
appVersion=appVersion, appName=appName)
self.create_file(filename, file_content)
def basic_dar_manifest_template(self):
xml_template = '<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n'
xml_template += ' <udm.DeploymentPackage version=\"$appVersion\" application=\"$appName\"> \n'
xml_template += ' <orchestrator /> \n'
xml_template += ' <deployables/> \n'
xml_template += ' </udm.DeploymentPackage> \n'
return Template(xml_template)
def package_dar(self, appName, appVersion):
command = "set -x \n"
command += "if [ -f %s_%s.dar ] \n"
command += " then rm -rf %s_%s.dar \n"
command += "fi \n"
command += "cd %s/%s \n" % (appVersion, appName)
command += "/usr/bin/zip -r %s/%s_%s.dar *" % (
self.workingDirectory, appName.replace(
'/', '_'), appVersion.replace('.', '_'))
self.execute_multiline_command(command)
def remove_dar(self, appName, appVersion):
command = "set -x \n"
command += "cd %s \n" % self.workingDirectory
command += "if [ -f %s_%s.dar ] \n" % (appName.replace(
'/', '_'), appVersion.replace('.', '_'))
command += " then rm -rf %s_%s.dar \n" % (appName.replace(
'/', '_'), appVersion.replace('.', '_'))
command += " fi \n"
self.execute_multiline_command(command)
def write_exec_script(self, commands, script):
self.session.upload_text_content_to_work_dir(self,
commands,
script,
executable=False)
def execute_multiline_command(self, command):
tmp_filename = self.filename_generator()
self.write_to_file(tmp_filename, command)
self.execute_command("chmod +x %s" % (tmp_filename))
self.execute_command("./%s" % (tmp_filename))
def filename_generator(self,
size=9,
chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def upload_dar_package(self, appName, appVersion, xldeployServer):
dar_file_name = "%s_%s.dar" % (appName.replace(
'/', '_'), appVersion.replace('.', '_'))
command = "/usr/bin/curl -k -u %s:%s -X POST -H \"content-type:multipart/form-data\" %s/package/upload/%s -F fileData=@./%s " % (
xldeployServer['username'], xldeployServer['password'],
xldeployServer['url'], dar_file_name, dar_file_name)
self.execute_command(command)
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp", work_directory=None):
# if 'key_file' in server:
# self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'])
# else:
# self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'],password=server['password'] )
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.session.execute("/bin/mkdir -p %s" % (self.WorkDirBase))
workDir = self.session.remote_file(self.WorkDirBase)
self.session.get_conn().setWorkingDirectory(workDir)
if work_directory is None:
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
else:
self.workDirectory = work_directory
self.RtcClient = server['pathToClientSoftware']
@staticmethod
def createServer(server, dirname, work_directory=None):
return ConfigRTCServer(server, dirname, work_directory)
def get_work_directory(self):
return self.workDirectory
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" % (self.getWorkDirectory(), file_name))
return response
def write_string_as_file(self, file_name, content):
remote_file = self.session.remote_file("%s/%s" % (self.getWorkDirectory(), file_name))
response = self.session.copy_text_to_file(str(content), remote_file)
def execute_lscm_command(self, command, run_in_workdirectory=False):
#logger.info(command)
command = self.sub_placeholders(command)
lscm_command="%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command_in_workDirectory(self, command):
# #logger.info("switching overthere to workdirectory")
# workDir = self.session.remote_file(self.get_work_directory())
# self.session.get_conn().setWorkingDirectory(workDir)
# #logger.info("switched to workDirectory")
command = "cd %s;%s" % (self.get_work_directory(), command)
return self.execute_command(command)
#def execute_command(self, command):
# #logger.info("executing command: %s " % (command))
# response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
# if response.rc != 0:
# #logger.info(response.stderr)
# #logger.info("unable to execute command %s" % (command))
# raise Exception('unable to execute command ')
# else:
# #logger.info("Response", str(response.stdout))
# return response
def execute_command(self, command, retain_scripts=True):
command_object = BashScriptBuilder()
for command_line in command.split(';'):
command_object.add_line(command_line)
executable_command = command_object.build()
tmp_script_file_name = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(9))
#logger.info("uploading script %s" % tmp_script_file_name)
script = self.session.upload_text_content_to_work_dir(executable_command, tmp_script_file_name, executable=True)
response = self.session.execute(script.getPath(), check_success=False, suppress_streaming_output=False)
#logger.info("stdErr for command yielded: " + str(response.stderr))
#logger.info("stdOut for command yielded: " + str(response.stdout))
#logger.info("the returncode for the command was:" + str(response.rc))
if response.rc != 0:
#logger.info("unable to execute command %s" % (command))
raise Exception('unable to execute command ')
else:
#logger.info("Response", str(response.stdout))
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.session.execute("/bin/mkdir -p %s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" % (self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
#logger.info(self.workDirectory)
#logger.info(self.getWorkDirectory())
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
class TestOverthereSession(object):
# preparing to test
def setUp(self):
self._linuxhost = OverthereHost(LocalConnectionOptions(os=OperatingSystemFamily.UNIX))
self._session = OverthereHostSession(self._linuxhost)
self._session.logger = OverthereSessionLogger(capture=True)
# ending the test
def tearDown(self):
self._session.close_conn()
def _clear_logs(self):
self._session.logger.output_lines = []
self._session.logger.error_lines = []
def assert_in_list(self, list, expect):
r = [s for s in list if str(s) == str(expect)]
eq_(len(r), 1, "expect [%s] to be in list [%s]" % (expect, str(list)))
def test_not_windows_host(self):
ok_(not self._session.is_windows())
def test_work_dir_cleanup(self):
workdir = self._session.work_dir()
ok_(workdir.exists())
workdir_path = workdir.path
self._session.close_conn()
ok_(not os.path.exists(workdir_path))
def test_read_write_file(self):
f = self._session.upload_text_content_to_work_dir("some text", "my.txt")
text = self._session.read_file(f.path)
eq_("some text", text)
def test_upload_executable_file(self):
f = self._session.upload_text_content_to_work_dir("#!/bin/sh\necho hello", "my.sh", executable=True)
r = self._session.execute(f.path)
self.assert_in_list(r.stdout, "hello")
def test_upload_classpath_resource(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/singleline.txt")
text = self._session.read_file(f.path)
eq_("some text 1", text)
def test_read_lines(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/multiline.txt")
lines = self._session.read_file_lines(f.path)
eq_(3, len(lines))
def test_execute(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
response = self._session.execute([f.path, "ping"])
eq_(response['rc'], 0)
eq_(response['stdout'][0], "Hi ping")
def test_execute_automatic_system_exit_on_failure(self):
success = False
try:
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
self._session.execute([f.path, "ping", "1"])
except SystemExit:
success = True
pass
eq_(success, True)
def test_execute_check_success_turned_off(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
response = self._session.execute("%s ping 1" % f.path, check_success=False)
eq_(response.rc, 1)
eq_(response['stdout'][0], "Hi ping")
eq_(response['stdout'][1], "Exiting with 1")
def test_with_support(self):
s = OverthereHostSession(self._linuxhost)
with s:
work_dir = s.work_dir().path
eq_(os.path.exists(work_dir), True)
eq_(os.path.exists(work_dir), False)
def _local_file(self, resource):
url = Thread.currentThread().contextClassLoader.getResource(resource)
return self._session.local_file(File(url.toURI()))
def test_mkdirs_on_dir_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
eq_(os.path.exists(target.path + "/echo.sh"), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_file_copy(self):
target = self._session.work_dir_file("some/path/some.txt")
source = self._local_file("testfiles/echo.sh")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.parentFile.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_turned_off_on_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
success = False
try:
self._session.copy_to(source, target, mkdirs=False)
except:
success = True
eq_(success, True)
def test_delete_from_when_target_does_not_exit(self):
target = self._session.work_dir_file("some/path")
source = None
self._session.delete_from(source, target)
log_info = self._session.logger.output_lines
eq_(len(log_info), 1)
eq_(log_info[0], "Target [%s] does not exist. No deletion to be performed" % target.path)
def test_delete_from_file_target(self):
target = self._session.work_dir_file("some/path/test.txt")
self._session.copy_text_to_file("xxx", target)
eq_(os.path.exists(target.path), True)
source = None
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
self.assert_in_list(log_info, "Deleting [%s]" % target.path)
def test_delete_from_folder_target(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(log_info, "Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting directory [%s]" % target.path)
def test_delete_from_with_external_resources(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
external_resource = self._session.work_dir_file("some/path/ext.txt")
self._session.copy_text_to_file("xxx", external_resource)
eq_(os.path.exists(external_resource.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(log_info, "Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Target directory [%s] is not shared, but still has content from an external source. Will not delete" % target.path)
def test_copy_diff(self):
old = self._local_file("directorycompare/old")
new = self._local_file("directorycompare/new")
deployed_old = self._session.work_dir_file("olddeployed")
self._session.copy_to(old, deployed_old)
eq_(os.path.exists(deployed_old.path), True)
self._clear_logs()
diff = Diff.calculate_diff(old, new)
self._session.copy_diff(deployed_old.path, diff)
log_info = self._session.logger.output_lines
eq_(len(log_info), 17)
self.assert_in_list(log_info, "3 files to be removed.")
self.assert_in_list(log_info, "3 new files to be copied.")
self.assert_in_list(log_info, "2 modified files to be copied.")
self.assert_in_list(log_info, "Start removal of files...")
self.assert_in_list(log_info, "Removing file %s/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removing directory %s/subdirremove" % deployed_old.path)
self.assert_in_list(log_info, "Removing file %s/subdirboth/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removal of files done.")
self.assert_in_list(log_info, "Start copying of new files...")
self.assert_in_list(log_info, "Copying directory %s/subdirnew" % deployed_old.path)
self.assert_in_list(log_info, "Copying file %s/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying file %s/subdirboth/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of new files done.")
self.assert_in_list(log_info, "Start copying of modified files...")
self.assert_in_list(log_info, "Updating file %s/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Updating file %s/subdirboth/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of modified files done.")
def test_execution_ctx_logging(self):
class ExecutionContext(object):
def __init__(self):
self.output_success = False
self.error_success = False
def logOutput(self, msg):
self.output_success = True
def logError(self, msg):
self.error_success = True
exec_ctx = ExecutionContext()
session = OverthereHostSession(self._linuxhost, execution_context=exec_ctx)
session.logger.info("Check")
eq_(exec_ctx.output_success, True)
session.logger.error("Check")
eq_(exec_ctx.error_success, True)
from rtc.RTCWorkspace import RTCWorkspace
rtc_client = repositoryService.read(request.query["client"])
rtc_repo = repositoryService.read(request.query["repo"])
sshOpts = SshConnectionOptions(rtc_client.getProperty("host"),
username=rtc_client.getProperty("username"),
password=rtc_client.getProperty("password"))
host = OverthereHost(sshOpts)
session = OverthereHostSession(host)
workspace = RTCWorkspace.createRTCWorkspace(rtc_repo, rtc_client)
loc = "/var/tmp/"
#Step one: change to a new directory to place configs
#TODO: Decide whether script makes directory or directory is there ahead of time
stepone = session.execute("cd " + loc, check_success=True)
#Retrieve key value of exported DAR
keyval = request.query["ciid"]
addkey = "curl -o " + loc + keyval + ".dar " + "http://*****:*****@localhost:4516/deployit/internal/download/" + keyval
#Step three: Execute cURL command to download DAR based on key to new directory
stepthree = session.execute(addkey, check_success=True)
getxml = "unzip " + loc + keyval + ".dar -d " + loc + keyval
#Step four: Unzip the DAR
stepfour = session.execute(getxml, check_success=True)
#SEnd the unzipped xml over to be filtered by helper class XmlFilter
def dump_events():
for pod in get_associated_pods(deployed):
print "Pod {0}".format(pod)
print "-------------------"
for event in get_pod_events(deployed, pod):
print event
attempts = 200
command_line = "{0} get {1} {2} -o=json".format(get_kubectl_command(deployed.container), resource, resourceName)
print command_line
session = OverthereHostSession(target_host)
try:
response = session.execute(command_line, suppress_streaming_output=True)
rc = response.rc
if rc != 0:
print "Non zero Exit Code {0}".format(rc)
result = "RETRY"
else:
data = json.loads(" ".join(response.stdout))
if not resourceName == "Deployment":
print data
else:
condition = data['status']['conditions'][0]
print "Status {status} {reason}: {message}".format(**condition)
availableReplicas = get_available_replicas(data)
replicas = 1 #deployed.replicas
print "availableReplicas {0}/ replicas {1}".format(availableReplicas, replicas)