def test_with_support(self):
s = OverthereHostSession(self._linuxhost)
with s:
work_dir = s.work_dir().path
eq_(os.path.exists(work_dir), True)
eq_(os.path.exists(work_dir), False)
def docker_machine_inspect(machine_name):
print "Inspect docker '{0}' machine ".format(machine_name)
command_line = "docker-machine inspect {0}".format(machine_name)
local_opts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(local_opts)
session = OverthereHostSession(host)
print "Executing '{0}'....".format(command_line)
try:
response = session.execute(command_line, check_success=False)
out = ""
for s in response.stdout:
out = "%s %s\n" % (out, s)
print out
for s in response.stderr:
print "ERR", s
if response.rc != 0:
sys.exit(response.rc)
return json.loads(out)
except:
tb = traceback.format_exc()
print "Error"
print tb
finally:
session.close_conn()
def preview(self,deployed):
try:
session = OverthereHostSession(self.helmclient.host,stream_command_output=False)
command_line = self.command_line(session,deployed)
print command_line
finally:
session.close_conn()
def test_with_support(self):
s = OverthereHostSession(self._linuxhost)
with s:
work_dir = s.work_dir().path
eq_(os.path.exists(work_dir), True)
eq_(os.path.exists(work_dir), False)
def content_to_file(input_content):
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(localOpts)
session = OverthereHostSession(host)
input_file = session.get_conn().getTempFile("a_json_file", ".json")
session.copy_text_to_file(input_content, input_file)
print("input_file is {0}".format(input_file))
return input_file
def _action(self, verb, data):
session = OverthereHostSession(self.cluster.kubectlHost)
data_str = self.data_to_string(data)
print data_str
remote_file = session.upload_text_content_to_work_dir(
data_str, 'k8s_{0}_resource.json'.format(verb))
command_line = "{0} {1} -f {2}".format(self.get_kubectl_command(),
verb, remote_file.path)
print command_line
self._execute(session, command_line)
def __init__(self, server, dirname="tmp"):
self.sshOpts = SshConnectionOptions(server['host'],
username=server['username'],
password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
self.RtcClient = server['pathToClientSoftware']
def get_associated_pods(ci):
pods = []
session = OverthereHostSession(target_host)
command_line = "{0} get {1} -l=component={2} -o json".format(get_kubectl_command(ci.container), 'pods', ci.name)
print command_line
response = session.execute(command_line)
if response.rc == 0:
data = json.loads(" ".join(response.stdout))
for item in data['items']:
pods.append(item['metadata']['name'])
return pods
def get_pod_events(ci, pod_name):
events = []
session = OverthereHostSession(target_host)
command_line = "{0} get {1} --field-selector involvedObject.name={2} -o json".format(
get_kubectl_command(ci.container), 'event',
pod_name)
response = session.execute(command_line)
if response.rc == 0:
data = json.loads(" ".join(response.stdout))
for item in data['items']:
events.append("{type} {message}".format(**item))
return events
def execute_script(server):
print "Running script [%s] on server instance [%s]" % (scriptToExecute, server.getProperty("serverName"))
sshOpts = SshConnectionOptions(server.getProperty("address"), server.getProperty("username"), password=server.getProperty("password"))
host = OverthereHost(sshOpts)
session = OverthereHostSession(host)
response = session.execute([scriptToExecute], check_success=False)
if response.rc != 0:
print "Failed to execute command"
print response.stderr
else:
print "Response", str(response.stdout)
session.close_conn()
class XLRunner:
def __init__(self, xlpath, server):
self.xlpath = xlpath
self.server = server
self.session = None
def get_xl_command(self):
options = '--xl-deploy-url {0} --xl-deploy-username {1} --xl-deploy-password {2}'.format(
self.server['url'], self.server['username'],
self.server['password'])
xl = "{0}/xl {1}".format(self.xlpath, options)
return xl
def parameters(self, content, values):
uploaded_file = self.session.upload_text_content_to_work_dir(
content, "content.yaml")
parameters = " -f " + uploaded_file.getPath()
if len(values) > 0:
parameters = parameters + " --values " + ",".join(
["{0}={1}".format(k, v) for k, v in values.items()])
return parameters
def command_line(self, content, values):
return "{0} apply {1}".format(self.get_xl_command(),
self.parameters(content, values))
def apply(self, content, values):
try:
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(localOpts)
self.session = OverthereHostSession(host,
stream_command_output=False)
command_line = self.command_line(content, values)
#print command_line
uploaded_runner = self.session.upload_text_content_to_work_dir(
command_line, 'xldeploy_xl_runner.sh', executable=True)
#print uploaded_runner.path
response = self.session.execute(command_line, check_success=False)
print "\n```"
print "\n ".join(response.stdout)
print "\n ".join(response.stderr)
print "```\n "
rc = response.rc
if response.rc > 0:
sys.exit(rc)
finally:
if not self.session == None:
self.session.close_conn()
class ConfigFsServer(object):
def __init__(self, server, username=None, password=None):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
def __del__(self):
self.session.close_conn()
@staticmethod
def createServer(server, username=None, password=None):
return ConfigFsServer(server, username, password)
def get_file_contents(self, file_path):
response = self.session.read_file(file_path)
return response
def __init__(self, server):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.zipBinary = server['pathToZipExecutable']
self.workingDirectory = server['workingDirectory']
self.create_directory(self.workingDirectory)
workDir = self.session.remote_file(self.workingDirectory)
self.session.get_conn().setWorkingDirectory(workDir)
def gke_describe(name, target):
command_line = "gcloud container clusters describe {0} --format=json".format(
name)
print command_line
host = target.container
session = OverthereHostSession(host)
print "Executing '{0}'....".format(command_line)
try:
response = session.execute(command_line)
return json.loads(" ".join(response.stdout))
except:
tb = traceback.format_exc()
print "Error"
print tb
finally:
session.close_conn()
def __init__(self, server, dirname="tmp"):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
self.RtcClient = server['pathToClientSoftware']
def docker_machine_env(machine_name):
print "Env docker '{0}' machine ".format(machine_name)
command_line = "docker-machine env {0}".format(machine_name)
local_opts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(local_opts)
session = OverthereHostSession(host)
print "Executing '{0}'....".format(command_line)
try:
response = session.execute(command_line)
return to_map(response.stdout)
except:
tb = traceback.format_exc()
print "Error"
print tb
finally:
session.close_conn()
def __init__(self, server):
self.retryCount = server['retryCount']
if self.retryCount == 0:
self.retryCount = 1
self.retryBaseInterval = 60
self.sshOpts = SshConnectionOptions(server['host'],
username=server['username'],
privateKeyFile=server['key_file'],
password=server['password'],
port=server['port'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
tries = 0
while tries < self.retryCount:
tries += 1
try:
self.zipBinary = server['pathToZipExecutable']
self.workingDirectory = server['workingDirectory']
self.create_directory(self.workingDirectory)
workDir = self.session.remote_file(self.workingDirectory)
self.session.get_conn().setWorkingDirectory(workDir)
break
except:
if tries > 3:
print sys.exc_info()[0]
print sys.exc_info()[1]
if tries > self.retryCount:
print sys.exc_info()[2].format_stack()
print "we were unable to setup a connection to server: %s after %i retries" % (
server['host'], tries)
sys.exit(2)
sleeptime = self.retryBaseInterval * tries
print "retrying to setup a connection to server %s in %i seconds" % (
server['host'], sleeptime)
time.sleep(sleeptime)
def run(self, verb, parameters, json_output=False, raise_on_fail=False):
session = OverthereHostSession(self.cluster.kubectlHost)
command_line = "{0} {1} {2}".format(self.get_kubectl_command(), verb,
parameters)
if json_output:
command_line = command_line + " -o json"
print command_line
response = session.execute(command_line,
suppress_streaming_output=True,
check_success=False)
if response.rc == 0:
stdout = "\n".join(response.stdout)
if json_output:
return json.loads(stdout)
else:
return stdout
else:
if raise_on_fail:
raise Exception("Kubectl Error when running '{0}':{1}".format(
command_line, '\n'.join(response.stderr)))
else:
return '\n'.join(response.stderr)
def apply(self, content, values):
try:
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(localOpts)
self.session = OverthereHostSession(host,
stream_command_output=False)
command_line = self.command_line(content, values)
#print command_line
uploaded_runner = self.session.upload_text_content_to_work_dir(
command_line, 'xldeploy_xl_runner.sh', executable=True)
#print uploaded_runner.path
response = self.session.execute(command_line, check_success=False)
print "\n```"
print "\n ".join(response.stdout)
print "\n ".join(response.stderr)
print "```\n "
rc = response.rc
if response.rc > 0:
sys.exit(rc)
finally:
if not self.session == None:
self.session.close_conn()
def execute(self, xmlaccessscript, config_uri=None):
session = OverthereHostSession(self.host, stream_command_output=False, execution_context=self.exec_context)
with session:
request_file = session.upload_text_content_to_work_dir(xmlaccessscript, "request.xml")
response_file = session.work_dir_file("response.xml")
fs = session.os.fileSeparator
executable = "%s%sbin%sxmlaccess%s" % (self.wp_home, fs, fs, session.os.scriptExtension)
wp_url = self.resolve_config_url(config_uri)
cmd_line = [executable, "-user", self.wp_user, "-password", self.wp_password, "-url", wp_url]
cmd_line.extend(["-in", request_file.path, "-out", response_file.path])
resp = session.execute(cmd_line, check_success=False)
response_xml = ""
if response_file.exists():
response_xml = session.read_file(response_file.path)
if resp.rc != 0:
self.exec_context.logError("XML Access failed!!!")
self.log_with_header("XML Access response:", response_xml, True)
self.log_with_header("Standard out", StringUtils.concat(resp.stdout))
self.log_with_header("Standard error", StringUtils.concat(resp.stderr))
cmd_line[4] = '******' # Password is 5th element in array
self.log_with_header("Executed command line", StringUtils.concat(cmd_line, " "))
sys.exit(1)
return response_xml
def test_execution_ctx_logging(self):
class ExecutionContext(object):
def __init__(self):
self.output_success = False
self.error_success = False
def logOutput(self, msg):
self.output_success = True
def logError(self, msg):
self.error_success = True
exec_ctx = ExecutionContext()
session = OverthereHostSession(self._linuxhost,
execution_context=exec_ctx)
session.logger.info("Check")
eq_(exec_ctx.output_success, True)
session.logger.error("Check")
eq_(exec_ctx.error_success, True)
def execute(self,deployed):
try:
session = OverthereHostSession(self.helmclient.host,stream_command_output=False)
command_line = self.command_line(session,deployed)
print command_line
uploaded_runner = session.upload_text_content_to_work_dir(command_line,'xldeploy_helm.sh',executable=True)
print uploaded_runner.path
response = session.execute(command_line,check_success=False)
print "\n ".join(response.stdout)
print "\n ".join(response.stderr)
rc = response.rc
if response.rc > 0:
sys.exit(rc)
finally:
session.close_conn()
def execute_docker_compose(context,
composed,
application,
command,
options=''):
session = OverthereHostSession(composed.container.host,
stream_command_output=True,
execution_context=context)
runtime = {
'application':
application,
'command':
command,
'options':
options,
'uploaded_compose_file':
session.upload_file_to_work_dir(composed.file),
'dockerHost':
composed.container.dockerHost,
'cert_pem':
session.upload_text_content_to_work_dir(composed.container.certPem,
"cert.pem"),
'ca_pem':
session.upload_text_content_to_work_dir(composed.container.caPem,
"ca.pem"),
'key_pem':
session.upload_text_content_to_work_dir(composed.container.keyPem,
"key.pem")
}
print "docker-compose {0}....".format(runtime['command'])
command_line = "docker-compose --file {r[uploaded_compose_file].path} --project-name {r[application]} --tlscacert {r[ca_pem].path} --tlscert {r[cert_pem].path} --tlskey {r[key_pem].path} --tlsverify -H {r[dockerHost]} --skip-hostname-check {r[command]} {r[options]} ".format(
r=runtime)
print '---------------------------'
print command_line
print '---------------------------'
response = session.execute(command_line)
if response.rc != 0:
sys.exit(response.rc)
def __init__(self, server):
self.retryCount = server['retryCount']
if self.retryCount == 0:
self.retryCount = 1
self.retryBaseInterval = 60
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'],password=server['password'],port=server['port'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
tries = 0
while tries < self.retryCount:
tries += 1
try:
self.zipBinary = server['pathToZipExecutable']
self.workingDirectory = server['workingDirectory']
self.create_directory(self.workingDirectory)
workDir = self.session.remote_file(self.workingDirectory)
self.session.get_conn().setWorkingDirectory(workDir)
break
except :
if tries > 3:
print sys.exc_info()[0]
print sys.exc_info()[1]
if tries > self.retryCount :
print sys.exc_info()[2].format_stack()
print "we were unable to setup a connection to server: %s after %i retries" % (server['host'], tries)
sys.exit(2)
sleeptime = self.retryBaseInterval * tries
print "retrying to setup a connection to server %s in %i seconds" % (server['host'], sleeptime)
time.sleep(sleeptime)
def __init__(self, server, dirname="tmp", work_directory=None):
# if 'key_file' in server:
# self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'])
# else:
# self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'],password=server['password'] )
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.session.execute("/bin/mkdir -p %s" % (self.WorkDirBase))
workDir = self.session.remote_file(self.WorkDirBase)
self.session.get_conn().setWorkingDirectory(workDir)
if work_directory is None:
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
else:
self.workDirectory = work_directory
self.RtcClient = server['pathToClientSoftware']
def TEST(deployed):
logger.error("======== TEST Diffs ===========")
ci = repositoryService.read(deployed)
for d in ci.deployeds:
dtype = str(d.type)
logger.error("dtype = " + dtype)
if dtype == "file.DeployedFile" or dtype == "file.Folder":
logger.error("Found a file object we can diff")
if dtype == "file.DeployedFile":
logger.error("Deal with files")
targetPath = str(d.targetPath)
logger.error("TargetPath = " + targetPath)
fileName = str(d.name)
logger.error("FileName = " + fileName)
logger.error("%s/%s " % (targetPath, fileName))
remote_path = "%s/%s" % (targetPath, fileName)
else:
logger.error("Deal with directories")
targetPath = repositoryService.read(d).targetPath
print "%s/ " % (targetPath)
remote_path = targetPath
# End if
logger.error("Connect to overthere host @ " + str(d.container))
with OverthereHostSession(d.container) as session:
diff_lines = remote_diff(session, fileName, remote_path)
logger.error("+--------------------------------------------")
logger.error("| %s on %s" % (d.name, d.container))
logger.error("+--------------------------------------------")
if len(diff_lines) > 0:
print "%s" % (diff_lines)
logger.error(StringUtils.concat(diff_lines))
print StringUtils.concat(diff_lines)
# End if
logger.error("+--------------------------------------------")
def execute(self, xmlaccessscript, config_uri=None):
session = OverthereHostSession(self.host,
stream_command_output=False,
execution_context=self.exec_context)
with session:
request_file = session.upload_text_content_to_work_dir(
xmlaccessscript, "request.xml")
response_file = session.work_dir_file("response.xml")
fs = session.os.fileSeparator
executable = "%s%sbin%sxmlaccess%s" % (self.wp_home, fs, fs,
session.os.scriptExtension)
wp_url = self.resolve_config_url(config_uri)
cmd_line = [
executable, "-user", self.wp_user, "-password",
self.wp_password, "-url", wp_url
]
cmd_line.extend(
["-in", request_file.path, "-out", response_file.path])
resp = session.execute(cmd_line, check_success=False)
response_xml = ""
if response_file.exists():
response_xml = session.read_file(response_file.path)
if resp.rc != 0:
self.exec_context.logError("XML Access failed!!!")
self.log_with_header("XML Access response:", response_xml,
True)
self.log_with_header("Standard out",
StringUtils.concat(resp.stdout))
self.log_with_header("Standard error",
StringUtils.concat(resp.stderr))
cmd_line[4] = '******' # Password is 5th element in array
self.log_with_header("Executed command line",
StringUtils.concat(cmd_line, " "))
sys.exit(1)
return response_xml
class DarBuildServer(object):
def __init__(self, server):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.zipBinary = server['pathToZipExecutable']
self.workingDirectory = server['workingDirectory']
self.create_directory(self.workingDirectory)
workDir = self.session.remote_file(self.workingDirectory)
self.session.get_conn().setWorkingDirectory(workDir)
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server):
return DarBuildServer(server)
#
def init_dar(self, appName, appVersion):
workspace_root = self.create_dar_directory(appName, appVersion)
manifest_filename = "%s/deployit-manifest.xml" % workspace_root
self.write_dar_manifest(appName, appVersion, workspace_root)
def import_ear(self, appName, appVersion, deployable, url):
self.download_file_into_dar(appName, appVersion,deployable, str(url))
def create_dar_directory(self, appName, appVersion):
dirName = "%s/%s" % (appVersion, appName)
# check if the directory exists .. if it does we should go do something else.
# might want to do a better locking mechanism
if self.dir_exists(dirName):
print "unable to create dar directory: %s/%s/%s" % (self.workingDirectory, appVersion, appName)
raise Exception('unable to create Dar Package Directory')
else:
self.create_directory(dirName)
return dirName
def download_file_into_dar(self, appName, appVersion, deployable, url):
#filename = url.split('/')[-1]
filename = os.path.basename(url)
outfile = "%s/%s/%s/%s" % (appVersion, appName,deployable, filename)
dirName = "%s/%s/%s" % (appVersion, appName,deployable)
if self.dir_exists(dirName):
print "output dir already exists: %s" % (dirName)
else:
self.create_directory(dirName)
self.execute_command("/usr/bin/curl -L -o %s %s" % (outfile, url))
def read_manifest(self, appName, appVersion):
file_name = "%s/%s/%s/deployit-manifest.xml" % (self.workingDirectory, appVersion, appName)
return self.session.read_file(file_name, encoding="UTF-8")
def write_manifest(self, appName, appVersion, content):
file_name = "%s/%s/deployit-manifest.xml" % (appVersion, appName)
self.write_to_file(file_name, content)
def create_directory(self, dirName):
self.execute_command("/bin/mkdir -p %s" % (dirName))
def create_file(self, fileName, content=None):
if content:
self.write_to_file(fileName, str(content))
else:
self.execute_command("/bin/touch %s" % (fileName))
def write_to_file(self, fileName, content):
remoteFile = self.session.remote_file("%s/%s" % (self.workingDirectory, fileName))
self.session.copy_text_to_file(str(content), remoteFile)
def dir_exists(self, dirName):
print dirName
command = "[ -d %s ]" % (dirName)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc == 0 :
return True
else:
return False
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response:", str(response.stdout)
# self.switch_working_directory()
return response
def write_dar_manifest(self, appName, appVersion, workspace_root):
filename = "./%s/deployit-manifest.xml" % (workspace_root)
file_content = self.basic_dar_manifest_template().substitute(appVersion=appVersion, appName=appName)
self.create_file(filename, file_content)
def basic_dar_manifest_template(self):
xml_template = '<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n'
xml_template += ' <udm.DeploymentPackage version=\"$appVersion\" application=\"$appName\"> \n'
xml_template += ' <orchestrator /> \n'
xml_template += ' <deployables/> \n'
xml_template += ' </udm.DeploymentPackage> \n'
return Template(xml_template)
def package_dar(self, appName, appVersion):
command = "if [ -f %s_%s.dar ] \n"
command += " then rm -rf %s_%s.dar \n"
command += "fi \n"
command += "cd %s/%s \n" % (appVersion, appName)
command += "/usr/bin/zip -r %s/%s_%s.dar *" % (self.workingDirectory, appName, appVersion.replace('.','_'))
self.execute_multiline_command(command)
def write_exec_script(self, commands, script):
self.session.upload_text_content_to_work_dir(self, commands, script, executable=False)
def execute_multiline_command(self, command):
tmp_filename = self.filename_generator()
self.write_to_file(tmp_filename, command)
self.execute_command("chmod +x %s" % (tmp_filename))
self.execute_command("./%s" % (tmp_filename))
def filename_generator(self, size=9, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def upload_dar_package(self, appName, appVersion, xldeployServer):
dar_file_name = "%s_%s.dar" % (appName, appVersion.replace('.','_'))
command = "/usr/bin/curl -u %s:%s -X POST -H \"content-type:multipart/form-data\" %s/package/upload/%s -F fileData=@./%s " % (xldeployServer['username'], xldeployServer['password'], xldeployServer['url'], dar_file_name, dar_file_name)
print command
self.execute_command(command)
template = Template(template_content)
values = {
'startDelay': smoketest.startDelay,
'maxRetries': smoketest.maxRetries,
'retryWaitInterval': smoketest.retryWaitInterval,
'target_url': context.getAttribute(deployed.id),
'expectedResponseText': smoketest.expectedResponseText
}
print values['target_url']
content = template.safe_substitute(values)
#print "-----------------"
#print content
#print "-----------------"
session = OverthereHostSession(target_host)
remote = session.upload_text_content_to_work_dir(content,
'benoit_aws_test.sh',
executable=True)
print remote.getPath()
response = session.execute(remote.getPath(), check_success=False)
for line in response.stdout:
print line
for line in response.stderr:
print line
rc = response.rc
if rc != 0:
print "Non zero Exit Code: {0}".format(rc)
sys.exit(rc)
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS
# FOR A PARTICULAR PURPOSE. THIS CODE AND INFORMATION ARE NOT SUPPORTED BY XEBIALABS.
#
from overtherepy import LocalConnectionOptions, OverthereHost, OverthereHostSession
from com.xebialabs.overthere import OperatingSystemFamily
import sys
import re
def to_map(stdout):
data = {}
for s in response.stdout:
if 'export' in s:
info = s.replace('export ', ' ').replace('"',
' ').strip().split('=')
data[str(info[0])] = str(info[1])
return data
session = OverthereHostSession(target.container.host)
response = session.execute("docker-machine env %s " % target.machineName)
data = to_map(response.stdout)
ip = re.findall(r'[0-9]+(?:\.[0-9]+){3}', data['DOCKER_HOST'])
if len(ip) == 1:
data['address'] = ip[0]
print "IP Address is %s " % data['address']
print data
session.close_conn()
context.setAttribute(context_key, data)
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp", work_directory=None):
# if 'key_file' in server:
# self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'])
# else:
# self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'],password=server['password'] )
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.session.execute("/bin/mkdir -p %s" % (self.WorkDirBase))
workDir = self.session.remote_file(self.WorkDirBase)
self.session.get_conn().setWorkingDirectory(workDir)
if work_directory is None:
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
else:
self.workDirectory = work_directory
self.RtcClient = server['pathToClientSoftware']
@staticmethod
def createServer(server, dirname, work_directory=None):
return ConfigRTCServer(server, dirname, work_directory)
def get_work_directory(self):
return self.workDirectory
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" % (self.getWorkDirectory(), file_name))
return response
def write_string_as_file(self, file_name, content):
remote_file = self.session.remote_file("%s/%s" % (self.getWorkDirectory(), file_name))
response = self.session.copy_text_to_file(str(content), remote_file)
def execute_lscm_command(self, command, run_in_workdirectory=False):
#logger.info(command)
command = self.sub_placeholders(command)
lscm_command="%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command_in_workDirectory(self, command):
# #logger.info("switching overthere to workdirectory")
# workDir = self.session.remote_file(self.get_work_directory())
# self.session.get_conn().setWorkingDirectory(workDir)
# #logger.info("switched to workDirectory")
command = "cd %s;%s" % (self.get_work_directory(), command)
return self.execute_command(command)
#def execute_command(self, command):
# #logger.info("executing command: %s " % (command))
# response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
# if response.rc != 0:
# #logger.info(response.stderr)
# #logger.info("unable to execute command %s" % (command))
# raise Exception('unable to execute command ')
# else:
# #logger.info("Response", str(response.stdout))
# return response
def execute_command(self, command, retain_scripts=True):
command_object = BashScriptBuilder()
for command_line in command.split(';'):
command_object.add_line(command_line)
executable_command = command_object.build()
tmp_script_file_name = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(9))
#logger.info("uploading script %s" % tmp_script_file_name)
script = self.session.upload_text_content_to_work_dir(executable_command, tmp_script_file_name, executable=True)
response = self.session.execute(script.getPath(), check_success=False, suppress_streaming_output=False)
#logger.info("stdErr for command yielded: " + str(response.stderr))
#logger.info("stdOut for command yielded: " + str(response.stdout))
#logger.info("the returncode for the command was:" + str(response.rc))
if response.rc != 0:
#logger.info("unable to execute command %s" % (command))
raise Exception('unable to execute command ')
else:
#logger.info("Response", str(response.stdout))
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.session.execute("/bin/mkdir -p %s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" % (self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
#logger.info(self.workDirectory)
#logger.info(self.getWorkDirectory())
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp"):
self.sshOpts = SshConnectionOptions(server['host'],
username=server['username'],
password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
self.RtcClient = server['pathToClientSoftware']
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server, dirname):
return ConfigRTCServer(server, dirname)
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" %
(self.getWorkDirectory(), file_name))
return response
def execute_lscm_command(self, command, run_in_workdirectory=False):
print command
command = self.sub_placeholders(command)
print command
lscm_command = "%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command,
check_success=False,
suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response", str(response.stdout)
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.execute_command(
"/bin/mkdir -p %s/%s/%s" %
(self.WorkDirBase, self.WorkDirTimeStamp, self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (
self.WorkDirBase, self.WorkDirTimeStamp, self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" %
(self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
print self.workDirectory
print self.getWorkDirectory()
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
class UnixBuildServer(object):
def __init__(self, server):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.base_working_directory = server['BaseWorkingDirectory']
self.working_directory = self.base_working_directory
self.create_working_directory(self.base_working_directory)
workDir = self.session.remote_file(self.base_working_directory)
self.session.get_conn().setWorkingDirectory(workDir)
@staticmethod
def createServer(server):
return UnixBuildServer(server)
def set_working_directory(self, relative_directory_path):
self.working_directory = "%s/%s" % (self.base_working_directory, relative_directory_path)
self.create_working_directory(self.working_directory)
workDir = self.session.remote_file(self.working_directory)
self.session.get_conn().setWorkingDirectory(workDir)
# main interface
# session related methods
def close_session(selfs):
self.session.close()
def create_working_directory(self, dir_full_path):
response = self.session.execute("/bin/mkdir -p %s" % (dir_full_path))
# File Handeling
def read_file_in_workspace(self, file_name):
print "not yet implemented"
def write_file_in_workspace(self, file_name, content):
print "not yet implemented"
def create_directory(self, directory_name):
self.execute_command(["/bin/mkdir -p %s" % (directory_name)])
# command Execution
def filename_generator(self, size=9, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def get_tmp_script_filename(self):
return "%s.sh" % self.filename_generator()
def execute_command(self, command, environment_variables=None):
# get a BashScriptBuilder object
command_object = BashScriptBuilder()
# copy in the environment variables
if environment_variables is not None:
for key, value in environment_variables.items():
command_object.set_env_var(key, value)
# add the rest of the possible multiline command
command_object.add_lines(command)
executable_command = command_object.build()
print "executing command: %s " % (executable_command)
tmp_script_filename = self.get_tmp_script_filename()
self.session.upload_text_content_to_work_dir(command_object.build(), tmp_script_filename, executable=True)
#
response = self.session.execute("%s/%s" % (self.working_directory, tmp_script_filename), check_success=False, suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print response.stdout
print "unable to execute command %s" % (command)
raise Exception('unable to execute command')
else:
print "Response:", str(response.stdout)
return response
#
# THIS CODE AND INFORMATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS
# FOR A PARTICULAR PURPOSE. THIS CODE AND INFORMATION ARE NOT SUPPORTED BY XEBIALABS.
#
from __future__ import with_statement
from overtherepy import OverthereHostSession
from liquibase import *
session = OverthereHostSession(container.host, stream_command_output=True, execution_context=context)
with session:
cmd_line = build_cmd_line(container)
cmd_line.extend(['tag', tag])
print_cmd_line(cmd_line, context)
session.execute(cmd_line)
class TestOverthereSession(object):
# preparing to test
def setUp(self):
self._linuxhost = OverthereHost(
LocalConnectionOptions(os=OperatingSystemFamily.UNIX))
self._session = OverthereHostSession(self._linuxhost)
self._session.logger = OverthereSessionLogger(capture=True)
# ending the test
def tearDown(self):
self._session.close_conn()
def _clear_logs(self):
self._session.logger.output_lines = []
self._session.logger.error_lines = []
def assert_in_list(self, list, expect):
r = [s for s in list if str(s) == str(expect)]
eq_(len(r), 1, "expect [%s] to be in list [%s]" % (expect, str(list)))
def test_not_windows_host(self):
ok_(not self._session.is_windows())
def test_work_dir_cleanup(self):
workdir = self._session.work_dir()
ok_(workdir.exists())
workdir_path = workdir.path
self._session.close_conn()
ok_(not os.path.exists(workdir_path))
def test_read_write_file(self):
f = self._session.upload_text_content_to_work_dir(
"some text", "my.txt")
text = self._session.read_file(f.path)
eq_("some text", text)
def test_upload_executable_file(self):
f = self._session.upload_text_content_to_work_dir(
"#!/bin/sh\necho hello", "my.sh", executable=True)
r = self._session.execute(f.path)
self.assert_in_list(r.stdout, "hello")
def test_upload_classpath_resource(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/singleline.txt")
text = self._session.read_file(f.path)
eq_("some text 1", text)
def test_read_lines(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/multiline.txt")
lines = self._session.read_file_lines(f.path)
eq_(3, len(lines))
def test_execute(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
response = self._session.execute([f.path, "ping"])
eq_(response['rc'], 0)
eq_(response['stdout'][0], "Hi ping")
def test_execute_automatic_system_exit_on_failure(self):
success = False
try:
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
self._session.execute([f.path, "ping", "1"])
except SystemExit:
success = True
pass
eq_(success, True)
def test_execute_check_success_turned_off(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
response = self._session.execute("%s ping 1" % f.path,
check_success=False)
eq_(response.rc, 1)
eq_(response['stdout'][0], "Hi ping")
eq_(response['stdout'][1], "Exiting with 1")
def test_with_support(self):
s = OverthereHostSession(self._linuxhost)
with s:
work_dir = s.work_dir().path
eq_(os.path.exists(work_dir), True)
eq_(os.path.exists(work_dir), False)
def _local_file(self, resource):
url = Thread.currentThread().contextClassLoader.getResource(resource)
return self._session.local_file(File(url.toURI()))
def test_mkdirs_on_dir_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
eq_(os.path.exists(target.path + "/echo.sh"), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_file_copy(self):
target = self._session.work_dir_file("some/path/some.txt")
source = self._local_file("testfiles/echo.sh")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.parentFile.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_turned_off_on_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
success = False
try:
self._session.copy_to(source, target, mkdirs=False)
except:
success = True
eq_(success, True)
def test_delete_from_when_target_does_not_exit(self):
target = self._session.work_dir_file("some/path")
source = None
self._session.delete_from(source, target)
log_info = self._session.logger.output_lines
eq_(len(log_info), 1)
eq_(
log_info[0],
"Target [%s] does not exist. No deletion to be performed" %
target.path)
def test_delete_from_file_target(self):
target = self._session.work_dir_file("some/path/test.txt")
self._session.copy_text_to_file("xxx", target)
eq_(os.path.exists(target.path), True)
source = None
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
self.assert_in_list(log_info, "Deleting [%s]" % target.path)
def test_delete_from_folder_target(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(
log_info,
"Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting directory [%s]" % target.path)
def test_delete_from_with_external_resources(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
external_resource = self._session.work_dir_file("some/path/ext.txt")
self._session.copy_text_to_file("xxx", external_resource)
eq_(os.path.exists(external_resource.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(
log_info,
"Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(
log_info,
"Target directory [%s] is not shared, but still has content from an external source. Will not delete"
% target.path)
def test_copy_diff(self):
old = self._local_file("directorycompare/old")
new = self._local_file("directorycompare/new")
deployed_old = self._session.work_dir_file("olddeployed")
self._session.copy_to(old, deployed_old)
eq_(os.path.exists(deployed_old.path), True)
self._clear_logs()
diff = Diff.calculate_diff(old, new)
self._session.copy_diff(deployed_old.path, diff)
log_info = self._session.logger.output_lines
eq_(len(log_info), 17)
self.assert_in_list(log_info, "3 files to be removed.")
self.assert_in_list(log_info, "3 new files to be copied.")
self.assert_in_list(log_info, "2 modified files to be copied.")
self.assert_in_list(log_info, "Start removal of files...")
self.assert_in_list(
log_info, "Removing file %s/removefile.txt" % deployed_old.path)
self.assert_in_list(
log_info, "Removing directory %s/subdirremove" % deployed_old.path)
self.assert_in_list(
log_info,
"Removing file %s/subdirboth/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removal of files done.")
self.assert_in_list(log_info, "Start copying of new files...")
self.assert_in_list(
log_info, "Copying directory %s/subdirnew" % deployed_old.path)
self.assert_in_list(log_info,
"Copying file %s/newfile.txt" % deployed_old.path)
self.assert_in_list(
log_info,
"Copying file %s/subdirboth/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of new files done.")
self.assert_in_list(log_info, "Start copying of modified files...")
self.assert_in_list(
log_info, "Updating file %s/changedfile.txt" % deployed_old.path)
self.assert_in_list(
log_info,
"Updating file %s/subdirboth/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of modified files done.")
def test_execution_ctx_logging(self):
class ExecutionContext(object):
def __init__(self):
self.output_success = False
self.error_success = False
def logOutput(self, msg):
self.output_success = True
def logError(self, msg):
self.error_success = True
exec_ctx = ExecutionContext()
session = OverthereHostSession(self._linuxhost,
execution_context=exec_ctx)
session.logger.info("Check")
eq_(exec_ctx.output_success, True)
session.logger.error("Check")
eq_(exec_ctx.error_success, True)
import java
import pprint
from overtherepy import SshConnectionOptions, OverthereHost, OverthereHostSession
from xldrtc import help
from rtc.RTCWorkspace import RTCWorkspace
cis = request.query['cis'].split(',')
logger.info(str(type(cis)))
logger.info(str(cis))
rtc_client = repositoryService.read(request.query["client"])
rtc_repo = repositoryService.read(request.query["repo"])
sshOpts = SshConnectionOptions(rtc_client.getProperty("host"),username = rtc_client.getProperty("username"), password = rtc_client.getProperty("password"))
host = OverthereHost(sshOpts)
session = OverthereHostSession(host)
workspace = RTCWorkspace.createRTCWorkspace(rtc_repo,rtc_client)
loc = "/var/tmp/"
#Step one: change to a new directory to place configs
#TODO: Decide whether script makes directory or directory is there ahead of time
stepone = session.execute("cd " + loc, check_success=True)
#Retrieve key value of exported DAR
keyval = request.query["ciid"]
addkey = "curl -o " + loc + keyval + ".dar " + "http://*****:*****@localhost:4516/deployit/internal/download/" + keyval
#Step three: Execute cURL command to download DAR based on key to new directory
stepthree = session.execute(addkey, check_success=True)
except:
return -1
attempts = 10
deployment_name = podname
if deployed.namespace is not None:
command_line = "{2} get deployment {0} --namespace={1} -o=json".format(
deployment_name, deployed.namespace, deployed.container.command)
else:
command_line = "{2} get deployment {0} -o=json".format(
deployment_name, deployed.namespace, deployed.container.command)
print command_line
try:
session = OverthereHostSession(deployed.container.host)
response = session.execute(command_line)
data = json.loads(" ".join(response.stdout))
condition = data['status']['conditions'][0]
print "Status {status} {reason}: {message}".format(**condition)
availableReplicas = get_available_replicas(data)
print "availableReplicas {0}/{1}".format(availableReplicas,
deployed.replicas)
if condition['status'] == "True":
print "Status Ok"
elif availableReplicas == deployed.replicas:
print "DONE replicas"
else:
inc_context()
#import helper class for XML filtering
import jarray
import java
import pprint
from overtherepy import SshConnectionOptions, OverthereHost, OverthereHostSession
from xldrtc import help
from rtc.RTCWorkspace import RTCWorkspace
rtc_client = repositoryService.read(request.query["client"])
rtc_repo = repositoryService.read(request.query["repo"])
sshOpts = SshConnectionOptions(rtc_client.getProperty("host"),
username=rtc_client.getProperty("username"),
password=rtc_client.getProperty("password"))
host = OverthereHost(sshOpts)
session = OverthereHostSession(host)
workspace = RTCWorkspace.createRTCWorkspace(rtc_repo, rtc_client)
loc = "/var/tmp/"
#Step one: change to a new directory to place configs
#TODO: Decide whether script makes directory or directory is there ahead of time
stepone = session.execute("cd " + loc, check_success=True)
#Retrieve key value of exported DAR
keyval = request.query["ciid"]
addkey = "curl -o " + loc + keyval + ".dar " + "http://*****:*****@localhost:4516/deployit/internal/download/" + keyval
#Step three: Execute cURL command to download DAR based on key to new directory
stepthree = session.execute(addkey, check_success=True)
def server_diff(hostA, hostB, source_file, remote_path):
global OverthereHostSession
global OperatingSystemFamily
global LocalConnectionOptions
global ObertherHostSession
global StringUtils
global Diff
#context.logOutput(remote_path)
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
local_session = OverthereHostSession(OverthereHost(localOpts))
sessionA = OverthereHostSession(hostA)
sessionB = OverthereHostSession(hostB)
try:
d1 = sessionA.remote_file(remote_path)
local_d1 = local_session.work_dir_file("hostA")
local_session.copy_to(d1, local_d1)
myFile = "%s" % (local_d1)
local_f1 = myFile.split(':')[1]
d2 = sessionB.remote_file(remote_path)
local_d2 = local_session.work_dir_file("hostB")
local_session.copy_to(d2, local_d2)
myFile = "%s" % (local_d2)
local_f2 = myFile.split(':')[1]
response = local_session.execute("diff %s %s" % (local_f1, local_f2), check_success=False)
finally:
sessionA.close_conn()
sessionB.close_conn()
local_session.close_conn()
# End try
return response.stdout
def setUp(self):
self._linuxhost = OverthereHost(
LocalConnectionOptions(os=OperatingSystemFamily.UNIX))
self._session = OverthereHostSession(self._linuxhost)
self._session.logger = OverthereSessionLogger(capture=True)
response = CommandResponse(rc=rc,
stdout=capture_so_handler.outputLines,
stderr=capture_se_handler.outputLines)
if response.rc != 0:
session.logger.error(StringUtils.concat(response.stdout))
session.logger.error(StringUtils.concat(response.stderr))
session.logger.error("Exit code {0}".format(response.rc))
sys.exit(response.rc)
return response
ansible_controler = repositoryService.read(ansible_controller_id)
remote_session = OverthereHostSession(ansibleController.host)
remote_yaml_file = remote_session.remote_file(yaml_file)
print("remote_yaml_file {0}".format(remote_yaml_file))
operating_system = System.getProperty("os.name").lower()
print("running " + operating_system + " OS on localhost")
if operating_system.startswith("win"):
os = OperatingSystemFamily.WINDOWS
else:
os = OperatingSystemFamily.UNIX
local_opts = LocalConnectionOptions(os=os)
local_host = OverthereHost(local_opts)
local_session = OverthereHostSession(local_host, stream_command_output=True)
local_yaml_file = local_session.work_dir_file(remote_yaml_file.getName())
#
# Copyright 2021 XEBIALABS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
from overtherepy import OverthereHostSession
session = OverthereHostSession(container.ansibleController.host)
target = session.remote_file(target_path)
print('Delete {0}'.format(target))
target.delete()
if container.container.kubeConfigContext is not None:
kubectl = kubectl + ' --context={0}'.format(deployed.container.container.kubeConfigContext)
return kubectl
def dump_events():
for pod in get_associated_pods(deployed):
print "Pod {0}".format(pod)
print "-------------------"
for event in get_pod_events(deployed, pod):
print event
attempts = 200
command_line = "{0} get {1} {2} -o=json".format(get_kubectl_command(deployed.container), resource, resourceName)
print command_line
session = OverthereHostSession(target_host)
try:
response = session.execute(command_line, suppress_streaming_output=True)
rc = response.rc
if rc != 0:
print "Non zero Exit Code {0}".format(rc)
result = "RETRY"
else:
data = json.loads(" ".join(response.stdout))
if not resourceName == "Deployment":
print data
else:
condition = data['status']['conditions'][0]
print "Status {status} {reason}: {message}".format(**condition)
availableReplicas = get_available_replicas(data)
import sys
import traceback
from com.xebialabs.overthere import OperatingSystemFamily
from overtherepy import LocalConnectionOptions, OverthereHost, OverthereHostSession
machine_name = thisCi.machineName
if machine_name is None:
raise ValueError("the 'Machine Name' property is empty")
print "Start docker '{0}' machine ".format(machine_name)
command_line = "docker-machine start {0}".format(machine_name)
localOpts = LocalConnectionOptions(os=OperatingSystemFamily.UNIX)
host = OverthereHost(localOpts)
session = OverthereHostSession(host)
print "Executing '{0}'....".format(command_line)
try:
response = session.execute(command_line, check_success=False)
for s in response.stdout:
print s
for s in response.stderr:
print s
except:
tb = traceback.format_exc()
print "Error"
print tb
finally:
session.close_conn()
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS
# FOR A PARTICULAR PURPOSE. THIS CODE AND INFORMATION ARE NOT SUPPORTED BY XEBIALABS.
#
from overtherepy import LocalConnectionOptions, OverthereHost, OverthereHostSession
from com.xebialabs.overthere import OperatingSystemFamily
import sys
import re
def to_map(stdout):
data={}
for s in response.stdout:
if 'export' in s:
info = s.replace('export ',' ').replace('"',' ').strip().split('=')
data[str(info[0])]=str(info[1])
return data
session = OverthereHostSession(target.container.host)
response = session.execute("docker-machine env %s " % target.machineName)
data=to_map(response.stdout)
ip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', data['DOCKER_HOST'])
if len(ip) == 1:
data['address'] = ip[0]
print "IP Address is %s " % data['address']
print data
session.close_conn()
context.setAttribute(context_key, data)
def __init__(self, server, username=None, password=None):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp"):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
self.RtcClient = server['pathToClientSoftware']
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server, dirname):
return ConfigRTCServer(server, dirname)
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" % (self.getWorkDirectory(), file_name))
return response
def execute_lscm_command(self, command, run_in_workdirectory=False):
print command
command = self.sub_placeholders(command)
print command
lscm_command="%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response", str(response.stdout)
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.execute_command("/bin/mkdir -p %s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" % (self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
print self.workDirectory
print self.getWorkDirectory()
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
class TestOverthereSession(object):
# preparing to test
def setUp(self):
self._linuxhost = OverthereHost(LocalConnectionOptions(os=OperatingSystemFamily.UNIX))
self._session = OverthereHostSession(self._linuxhost)
self._session.logger = OverthereSessionLogger(capture=True)
# ending the test
def tearDown(self):
self._session.close_conn()
def _clear_logs(self):
self._session.logger.output_lines = []
self._session.logger.error_lines = []
def assert_in_list(self, list, expect):
r = [s for s in list if str(s) == str(expect)]
eq_(len(r), 1, "expect [%s] to be in list [%s]" % (expect, str(list)))
def test_not_windows_host(self):
ok_(not self._session.is_windows())
def test_work_dir_cleanup(self):
workdir = self._session.work_dir()
ok_(workdir.exists())
workdir_path = workdir.path
self._session.close_conn()
ok_(not os.path.exists(workdir_path))
def test_read_write_file(self):
f = self._session.upload_text_content_to_work_dir("some text", "my.txt")
text = self._session.read_file(f.path)
eq_("some text", text)
def test_upload_executable_file(self):
f = self._session.upload_text_content_to_work_dir("#!/bin/sh\necho hello", "my.sh", executable=True)
r = self._session.execute(f.path)
self.assert_in_list(r.stdout, "hello")
def test_upload_classpath_resource(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/singleline.txt")
text = self._session.read_file(f.path)
eq_("some text 1", text)
def test_read_lines(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/multiline.txt")
lines = self._session.read_file_lines(f.path)
eq_(3, len(lines))
def test_execute(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
response = self._session.execute([f.path, "ping"])
eq_(response['rc'], 0)
eq_(response['stdout'][0], "Hi ping")
def test_execute_automatic_system_exit_on_failure(self):
success = False
try:
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
self._session.execute([f.path, "ping", "1"])
except SystemExit:
success = True
pass
eq_(success, True)
def test_execute_check_success_turned_off(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
response = self._session.execute("%s ping 1" % f.path, check_success=False)
eq_(response.rc, 1)
eq_(response['stdout'][0], "Hi ping")
eq_(response['stdout'][1], "Exiting with 1")
def test_with_support(self):
s = OverthereHostSession(self._linuxhost)
with s:
work_dir = s.work_dir().path
eq_(os.path.exists(work_dir), True)
eq_(os.path.exists(work_dir), False)
def _local_file(self, resource):
url = Thread.currentThread().contextClassLoader.getResource(resource)
return self._session.local_file(File(url.toURI()))
def test_mkdirs_on_dir_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
eq_(os.path.exists(target.path + "/echo.sh"), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_file_copy(self):
target = self._session.work_dir_file("some/path/some.txt")
source = self._local_file("testfiles/echo.sh")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.parentFile.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_turned_off_on_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
success = False
try:
self._session.copy_to(source, target, mkdirs=False)
except:
success = True
eq_(success, True)
def test_delete_from_when_target_does_not_exit(self):
target = self._session.work_dir_file("some/path")
source = None
self._session.delete_from(source, target)
log_info = self._session.logger.output_lines
eq_(len(log_info), 1)
eq_(log_info[0], "Target [%s] does not exist. No deletion to be performed" % target.path)
def test_delete_from_file_target(self):
target = self._session.work_dir_file("some/path/test.txt")
self._session.copy_text_to_file("xxx", target)
eq_(os.path.exists(target.path), True)
source = None
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
self.assert_in_list(log_info, "Deleting [%s]" % target.path)
def test_delete_from_folder_target(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(log_info, "Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting directory [%s]" % target.path)
def test_delete_from_with_external_resources(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
external_resource = self._session.work_dir_file("some/path/ext.txt")
self._session.copy_text_to_file("xxx", external_resource)
eq_(os.path.exists(external_resource.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(log_info, "Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Target directory [%s] is not shared, but still has content from an external source. Will not delete" % target.path)
def test_copy_diff(self):
old = self._local_file("directorycompare/old")
new = self._local_file("directorycompare/new")
deployed_old = self._session.work_dir_file("olddeployed")
self._session.copy_to(old, deployed_old)
eq_(os.path.exists(deployed_old.path), True)
self._clear_logs()
diff = Diff.calculate_diff(old, new)
self._session.copy_diff(deployed_old.path, diff)
log_info = self._session.logger.output_lines
eq_(len(log_info), 17)
self.assert_in_list(log_info, "3 files to be removed.")
self.assert_in_list(log_info, "3 new files to be copied.")
self.assert_in_list(log_info, "2 modified files to be copied.")
self.assert_in_list(log_info, "Start removal of files...")
self.assert_in_list(log_info, "Removing file %s/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removing directory %s/subdirremove" % deployed_old.path)
self.assert_in_list(log_info, "Removing file %s/subdirboth/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removal of files done.")
self.assert_in_list(log_info, "Start copying of new files...")
self.assert_in_list(log_info, "Copying directory %s/subdirnew" % deployed_old.path)
self.assert_in_list(log_info, "Copying file %s/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying file %s/subdirboth/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of new files done.")
self.assert_in_list(log_info, "Start copying of modified files...")
self.assert_in_list(log_info, "Updating file %s/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Updating file %s/subdirboth/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of modified files done.")
def test_execution_ctx_logging(self):
class ExecutionContext(object):
def __init__(self):
self.output_success = False
self.error_success = False
def logOutput(self, msg):
self.output_success = True
def logError(self, msg):
self.error_success = True
exec_ctx = ExecutionContext()
session = OverthereHostSession(self._linuxhost, execution_context=exec_ctx)
session.logger.info("Check")
eq_(exec_ctx.output_success, True)
session.logger.error("Check")
eq_(exec_ctx.error_success, True)
def setUp(self):
self._linuxhost = OverthereHost(LocalConnectionOptions(os=OperatingSystemFamily.UNIX))
self._session = OverthereHostSession(self._linuxhost)
self._session.logger = OverthereSessionLogger(capture=True)
