class ConfigFsServer(object):
def __init__(self, server, username=None, password=None):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
def __del__(self):
self.session.close_conn()
@staticmethod
def createServer(server, username=None, password=None):
return ConfigFsServer(server, username, password)
def get_file_contents(self, file_path):
response = self.session.read_file(file_path)
return response
def execute(self, xmlaccessscript, config_uri=None):
session = OverthereHostSession(self.host, stream_command_output=False, execution_context=self.exec_context)
with session:
request_file = session.upload_text_content_to_work_dir(xmlaccessscript, "request.xml")
response_file = session.work_dir_file("response.xml")
fs = session.os.fileSeparator
executable = "%s%sbin%sxmlaccess%s" % (self.wp_home, fs, fs, session.os.scriptExtension)
wp_url = self.resolve_config_url(config_uri)
cmd_line = [executable, "-user", self.wp_user, "-password", self.wp_password, "-url", wp_url]
cmd_line.extend(["-in", request_file.path, "-out", response_file.path])
resp = session.execute(cmd_line, check_success=False)
response_xml = ""
if response_file.exists():
response_xml = session.read_file(response_file.path)
if resp.rc != 0:
self.exec_context.logError("XML Access failed!!!")
self.log_with_header("XML Access response:", response_xml, True)
self.log_with_header("Standard out", StringUtils.concat(resp.stdout))
self.log_with_header("Standard error", StringUtils.concat(resp.stderr))
cmd_line[4] = '******' # Password is 5th element in array
self.log_with_header("Executed command line", StringUtils.concat(cmd_line, " "))
sys.exit(1)
return response_xml
def execute(self, xmlaccessscript, config_uri=None):
session = OverthereHostSession(self.host,
stream_command_output=False,
execution_context=self.exec_context)
with session:
request_file = session.upload_text_content_to_work_dir(
xmlaccessscript, "request.xml")
response_file = session.work_dir_file("response.xml")
fs = session.os.fileSeparator
executable = "%s%sbin%sxmlaccess%s" % (self.wp_home, fs, fs,
session.os.scriptExtension)
wp_url = self.resolve_config_url(config_uri)
cmd_line = [
executable, "-user", self.wp_user, "-password",
self.wp_password, "-url", wp_url
]
cmd_line.extend(
["-in", request_file.path, "-out", response_file.path])
resp = session.execute(cmd_line, check_success=False)
response_xml = ""
if response_file.exists():
response_xml = session.read_file(response_file.path)
if resp.rc != 0:
self.exec_context.logError("XML Access failed!!!")
self.log_with_header("XML Access response:", response_xml,
True)
self.log_with_header("Standard out",
StringUtils.concat(resp.stdout))
self.log_with_header("Standard error",
StringUtils.concat(resp.stderr))
cmd_line[4] = '******' # Password is 5th element in array
self.log_with_header("Executed command line",
StringUtils.concat(cmd_line, " "))
sys.exit(1)
return response_xml
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp"):
self.sshOpts = SshConnectionOptions(server['host'],
username=server['username'],
password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
self.RtcClient = server['pathToClientSoftware']
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server, dirname):
return ConfigRTCServer(server, dirname)
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" %
(self.getWorkDirectory(), file_name))
return response
def execute_lscm_command(self, command, run_in_workdirectory=False):
print command
command = self.sub_placeholders(command)
print command
lscm_command = "%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command,
check_success=False,
suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response", str(response.stdout)
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.execute_command(
"/bin/mkdir -p %s/%s/%s" %
(self.WorkDirBase, self.WorkDirTimeStamp, self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (
self.WorkDirBase, self.WorkDirTimeStamp, self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" %
(self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
print self.workDirectory
print self.getWorkDirectory()
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
class DarBuildServer(object):
def __init__(self, server):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.zipBinary = server['pathToZipExecutable']
self.workingDirectory = server['workingDirectory']
self.create_directory(self.workingDirectory)
workDir = self.session.remote_file(self.workingDirectory)
self.session.get_conn().setWorkingDirectory(workDir)
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server):
return DarBuildServer(server)
#
def init_dar(self, appName, appVersion):
workspace_root = self.create_dar_directory(appName, appVersion)
manifest_filename = "%s/deployit-manifest.xml" % workspace_root
self.write_dar_manifest(appName, appVersion, workspace_root)
def import_ear(self, appName, appVersion, deployable, url):
self.download_file_into_dar(appName, appVersion,deployable, str(url))
def create_dar_directory(self, appName, appVersion):
dirName = "%s/%s" % (appVersion, appName)
# check if the directory exists .. if it does we should go do something else.
# might want to do a better locking mechanism
if self.dir_exists(dirName):
print "unable to create dar directory: %s/%s/%s" % (self.workingDirectory, appVersion, appName)
raise Exception('unable to create Dar Package Directory')
else:
self.create_directory(dirName)
return dirName
def download_file_into_dar(self, appName, appVersion, deployable, url):
#filename = url.split('/')[-1]
filename = os.path.basename(url)
outfile = "%s/%s/%s/%s" % (appVersion, appName,deployable, filename)
dirName = "%s/%s/%s" % (appVersion, appName,deployable)
if self.dir_exists(dirName):
print "output dir already exists: %s" % (dirName)
else:
self.create_directory(dirName)
self.execute_command("/usr/bin/curl -L -o %s %s" % (outfile, url))
def read_manifest(self, appName, appVersion):
file_name = "%s/%s/%s/deployit-manifest.xml" % (self.workingDirectory, appVersion, appName)
return self.session.read_file(file_name, encoding="UTF-8")
def write_manifest(self, appName, appVersion, content):
file_name = "%s/%s/deployit-manifest.xml" % (appVersion, appName)
self.write_to_file(file_name, content)
def create_directory(self, dirName):
self.execute_command("/bin/mkdir -p %s" % (dirName))
def create_file(self, fileName, content=None):
if content:
self.write_to_file(fileName, str(content))
else:
self.execute_command("/bin/touch %s" % (fileName))
def write_to_file(self, fileName, content):
remoteFile = self.session.remote_file("%s/%s" % (self.workingDirectory, fileName))
self.session.copy_text_to_file(str(content), remoteFile)
def dir_exists(self, dirName):
print dirName
command = "[ -d %s ]" % (dirName)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc == 0 :
return True
else:
return False
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response:", str(response.stdout)
# self.switch_working_directory()
return response
def write_dar_manifest(self, appName, appVersion, workspace_root):
filename = "./%s/deployit-manifest.xml" % (workspace_root)
file_content = self.basic_dar_manifest_template().substitute(appVersion=appVersion, appName=appName)
self.create_file(filename, file_content)
def basic_dar_manifest_template(self):
xml_template = '<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n'
xml_template += ' <udm.DeploymentPackage version=\"$appVersion\" application=\"$appName\"> \n'
xml_template += ' <orchestrator /> \n'
xml_template += ' <deployables/> \n'
xml_template += ' </udm.DeploymentPackage> \n'
return Template(xml_template)
def package_dar(self, appName, appVersion):
command = "if [ -f %s_%s.dar ] \n"
command += " then rm -rf %s_%s.dar \n"
command += "fi \n"
command += "cd %s/%s \n" % (appVersion, appName)
command += "/usr/bin/zip -r %s/%s_%s.dar *" % (self.workingDirectory, appName, appVersion.replace('.','_'))
self.execute_multiline_command(command)
def write_exec_script(self, commands, script):
self.session.upload_text_content_to_work_dir(self, commands, script, executable=False)
def execute_multiline_command(self, command):
tmp_filename = self.filename_generator()
self.write_to_file(tmp_filename, command)
self.execute_command("chmod +x %s" % (tmp_filename))
self.execute_command("./%s" % (tmp_filename))
def filename_generator(self, size=9, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def upload_dar_package(self, appName, appVersion, xldeployServer):
dar_file_name = "%s_%s.dar" % (appName, appVersion.replace('.','_'))
command = "/usr/bin/curl -u %s:%s -X POST -H \"content-type:multipart/form-data\" %s/package/upload/%s -F fileData=@./%s " % (xldeployServer['username'], xldeployServer['password'], xldeployServer['url'], dar_file_name, dar_file_name)
print command
self.execute_command(command)
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp"):
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
self.RtcClient = server['pathToClientSoftware']
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
@staticmethod
def createServer(server, dirname):
return ConfigRTCServer(server, dirname)
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" % (self.getWorkDirectory(), file_name))
return response
def execute_lscm_command(self, command, run_in_workdirectory=False):
print command
command = self.sub_placeholders(command)
print command
lscm_command="%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response", str(response.stdout)
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.execute_command("/bin/mkdir -p %s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" % (self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
print self.workDirectory
print self.getWorkDirectory()
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
class TestOverthereSession(object):
# preparing to test
def setUp(self):
self._linuxhost = OverthereHost(LocalConnectionOptions(os=OperatingSystemFamily.UNIX))
self._session = OverthereHostSession(self._linuxhost)
self._session.logger = OverthereSessionLogger(capture=True)
# ending the test
def tearDown(self):
self._session.close_conn()
def _clear_logs(self):
self._session.logger.output_lines = []
self._session.logger.error_lines = []
def assert_in_list(self, list, expect):
r = [s for s in list if str(s) == str(expect)]
eq_(len(r), 1, "expect [%s] to be in list [%s]" % (expect, str(list)))
def test_not_windows_host(self):
ok_(not self._session.is_windows())
def test_work_dir_cleanup(self):
workdir = self._session.work_dir()
ok_(workdir.exists())
workdir_path = workdir.path
self._session.close_conn()
ok_(not os.path.exists(workdir_path))
def test_read_write_file(self):
f = self._session.upload_text_content_to_work_dir("some text", "my.txt")
text = self._session.read_file(f.path)
eq_("some text", text)
def test_upload_executable_file(self):
f = self._session.upload_text_content_to_work_dir("#!/bin/sh\necho hello", "my.sh", executable=True)
r = self._session.execute(f.path)
self.assert_in_list(r.stdout, "hello")
def test_upload_classpath_resource(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/singleline.txt")
text = self._session.read_file(f.path)
eq_("some text 1", text)
def test_read_lines(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/multiline.txt")
lines = self._session.read_file_lines(f.path)
eq_(3, len(lines))
def test_execute(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
response = self._session.execute([f.path, "ping"])
eq_(response['rc'], 0)
eq_(response['stdout'][0], "Hi ping")
def test_execute_automatic_system_exit_on_failure(self):
success = False
try:
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
self._session.execute([f.path, "ping", "1"])
except SystemExit:
success = True
pass
eq_(success, True)
def test_execute_check_success_turned_off(self):
f = self._session.upload_classpath_resource_to_work_dir("testfiles/echo.sh", executable=True)
response = self._session.execute("%s ping 1" % f.path, check_success=False)
eq_(response.rc, 1)
eq_(response['stdout'][0], "Hi ping")
eq_(response['stdout'][1], "Exiting with 1")
def test_with_support(self):
s = OverthereHostSession(self._linuxhost)
with s:
work_dir = s.work_dir().path
eq_(os.path.exists(work_dir), True)
eq_(os.path.exists(work_dir), False)
def _local_file(self, resource):
url = Thread.currentThread().contextClassLoader.getResource(resource)
return self._session.local_file(File(url.toURI()))
def test_mkdirs_on_dir_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
eq_(os.path.exists(target.path + "/echo.sh"), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_file_copy(self):
target = self._session.work_dir_file("some/path/some.txt")
source = self._local_file("testfiles/echo.sh")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.parentFile.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_turned_off_on_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
success = False
try:
self._session.copy_to(source, target, mkdirs=False)
except:
success = True
eq_(success, True)
def test_delete_from_when_target_does_not_exit(self):
target = self._session.work_dir_file("some/path")
source = None
self._session.delete_from(source, target)
log_info = self._session.logger.output_lines
eq_(len(log_info), 1)
eq_(log_info[0], "Target [%s] does not exist. No deletion to be performed" % target.path)
def test_delete_from_file_target(self):
target = self._session.work_dir_file("some/path/test.txt")
self._session.copy_text_to_file("xxx", target)
eq_(os.path.exists(target.path), True)
source = None
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
self.assert_in_list(log_info, "Deleting [%s]" % target.path)
def test_delete_from_folder_target(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(log_info, "Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting directory [%s]" % target.path)
def test_delete_from_with_external_resources(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
external_resource = self._session.work_dir_file("some/path/ext.txt")
self._session.copy_text_to_file("xxx", external_resource)
eq_(os.path.exists(external_resource.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(log_info, "Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Target directory [%s] is not shared, but still has content from an external source. Will not delete" % target.path)
def test_copy_diff(self):
old = self._local_file("directorycompare/old")
new = self._local_file("directorycompare/new")
deployed_old = self._session.work_dir_file("olddeployed")
self._session.copy_to(old, deployed_old)
eq_(os.path.exists(deployed_old.path), True)
self._clear_logs()
diff = Diff.calculate_diff(old, new)
self._session.copy_diff(deployed_old.path, diff)
log_info = self._session.logger.output_lines
eq_(len(log_info), 17)
self.assert_in_list(log_info, "3 files to be removed.")
self.assert_in_list(log_info, "3 new files to be copied.")
self.assert_in_list(log_info, "2 modified files to be copied.")
self.assert_in_list(log_info, "Start removal of files...")
self.assert_in_list(log_info, "Removing file %s/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removing directory %s/subdirremove" % deployed_old.path)
self.assert_in_list(log_info, "Removing file %s/subdirboth/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removal of files done.")
self.assert_in_list(log_info, "Start copying of new files...")
self.assert_in_list(log_info, "Copying directory %s/subdirnew" % deployed_old.path)
self.assert_in_list(log_info, "Copying file %s/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying file %s/subdirboth/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of new files done.")
self.assert_in_list(log_info, "Start copying of modified files...")
self.assert_in_list(log_info, "Updating file %s/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Updating file %s/subdirboth/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of modified files done.")
def test_execution_ctx_logging(self):
class ExecutionContext(object):
def __init__(self):
self.output_success = False
self.error_success = False
def logOutput(self, msg):
self.output_success = True
def logError(self, msg):
self.error_success = True
exec_ctx = ExecutionContext()
session = OverthereHostSession(self._linuxhost, execution_context=exec_ctx)
session.logger.info("Check")
eq_(exec_ctx.output_success, True)
session.logger.error("Check")
eq_(exec_ctx.error_success, True)
class ConfigRTCServer(object):
def __init__(self, server, dirname="tmp", work_directory=None):
# if 'key_file' in server:
# self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'])
# else:
# self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],password=server['password'])
self.sshOpts = SshConnectionOptions(server['host'], username=server['username'],privateKeyFile=server['key_file'],password=server['password'] )
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
self.WorkDirBase = server['filesystemLocation']
self.session.execute("/bin/mkdir -p %s" % (self.WorkDirBase))
workDir = self.session.remote_file(self.WorkDirBase)
self.session.get_conn().setWorkingDirectory(workDir)
if work_directory is None:
self.WorkDirTimeStamp = int(time.time())
self.WorkDirName = dirname
self.workDirectory = None
else:
self.workDirectory = work_directory
self.RtcClient = server['pathToClientSoftware']
@staticmethod
def createServer(server, dirname, work_directory=None):
return ConfigRTCServer(server, dirname, work_directory)
def get_work_directory(self):
return self.workDirectory
def get_file_contents(self, file_name):
response = self.session.read_file("%s/%s" % (self.getWorkDirectory(), file_name))
return response
def write_string_as_file(self, file_name, content):
remote_file = self.session.remote_file("%s/%s" % (self.getWorkDirectory(), file_name))
response = self.session.copy_text_to_file(str(content), remote_file)
def execute_lscm_command(self, command, run_in_workdirectory=False):
#logger.info(command)
command = self.sub_placeholders(command)
lscm_command="%s %s" % (self.RtcClient, command)
if run_in_workdirectory is False:
response = self.execute_command(lscm_command)
else:
response = self.execute_command_in_workDirectory(lscm_command)
return response
def execute_command_in_workDirectory(self, command):
# #logger.info("switching overthere to workdirectory")
# workDir = self.session.remote_file(self.get_work_directory())
# self.session.get_conn().setWorkingDirectory(workDir)
# #logger.info("switched to workDirectory")
command = "cd %s;%s" % (self.get_work_directory(), command)
return self.execute_command(command)
#def execute_command(self, command):
# #logger.info("executing command: %s " % (command))
# response = self.session.execute(command, check_success=False, suppress_streaming_output=False)
# if response.rc != 0:
# #logger.info(response.stderr)
# #logger.info("unable to execute command %s" % (command))
# raise Exception('unable to execute command ')
# else:
# #logger.info("Response", str(response.stdout))
# return response
def execute_command(self, command, retain_scripts=True):
command_object = BashScriptBuilder()
for command_line in command.split(';'):
command_object.add_line(command_line)
executable_command = command_object.build()
tmp_script_file_name = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(9))
#logger.info("uploading script %s" % tmp_script_file_name)
script = self.session.upload_text_content_to_work_dir(executable_command, tmp_script_file_name, executable=True)
response = self.session.execute(script.getPath(), check_success=False, suppress_streaming_output=False)
#logger.info("stdErr for command yielded: " + str(response.stderr))
#logger.info("stdOut for command yielded: " + str(response.stdout))
#logger.info("the returncode for the command was:" + str(response.rc))
if response.rc != 0:
#logger.info("unable to execute command %s" % (command))
raise Exception('unable to execute command ')
else:
#logger.info("Response", str(response.stdout))
return response
def getWorkDirectory(self):
if self.workDirectory is None:
self.session.execute("/bin/mkdir -p %s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName))
self.workDirectory = "%s/%s/%s" % (self.WorkDirBase,self.WorkDirTimeStamp,self.WorkDirName)
return self.workDirectory
def destroy_work_directory(self):
if self.workDirectory is not None:
self.execute_command("/bin/rm -rf %s/%s" % (self.WorkDirBase, self.WorkDirName))
def sub_placeholders(self, input):
#logger.info(self.workDirectory)
#logger.info(self.getWorkDirectory())
output = input.replace('<work_dir>', self.getWorkDirectory())
return output
class TestOverthereSession(object):
# preparing to test
def setUp(self):
self._linuxhost = OverthereHost(
LocalConnectionOptions(os=OperatingSystemFamily.UNIX))
self._session = OverthereHostSession(self._linuxhost)
self._session.logger = OverthereSessionLogger(capture=True)
# ending the test
def tearDown(self):
self._session.close_conn()
def _clear_logs(self):
self._session.logger.output_lines = []
self._session.logger.error_lines = []
def assert_in_list(self, list, expect):
r = [s for s in list if str(s) == str(expect)]
eq_(len(r), 1, "expect [%s] to be in list [%s]" % (expect, str(list)))
def test_not_windows_host(self):
ok_(not self._session.is_windows())
def test_work_dir_cleanup(self):
workdir = self._session.work_dir()
ok_(workdir.exists())
workdir_path = workdir.path
self._session.close_conn()
ok_(not os.path.exists(workdir_path))
def test_read_write_file(self):
f = self._session.upload_text_content_to_work_dir(
"some text", "my.txt")
text = self._session.read_file(f.path)
eq_("some text", text)
def test_upload_executable_file(self):
f = self._session.upload_text_content_to_work_dir(
"#!/bin/sh\necho hello", "my.sh", executable=True)
r = self._session.execute(f.path)
self.assert_in_list(r.stdout, "hello")
def test_upload_classpath_resource(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/singleline.txt")
text = self._session.read_file(f.path)
eq_("some text 1", text)
def test_read_lines(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/multiline.txt")
lines = self._session.read_file_lines(f.path)
eq_(3, len(lines))
def test_execute(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
response = self._session.execute([f.path, "ping"])
eq_(response['rc'], 0)
eq_(response['stdout'][0], "Hi ping")
def test_execute_automatic_system_exit_on_failure(self):
success = False
try:
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
self._session.execute([f.path, "ping", "1"])
except SystemExit:
success = True
pass
eq_(success, True)
def test_execute_check_success_turned_off(self):
f = self._session.upload_classpath_resource_to_work_dir(
"testfiles/echo.sh", executable=True)
response = self._session.execute("%s ping 1" % f.path,
check_success=False)
eq_(response.rc, 1)
eq_(response['stdout'][0], "Hi ping")
eq_(response['stdout'][1], "Exiting with 1")
def test_with_support(self):
s = OverthereHostSession(self._linuxhost)
with s:
work_dir = s.work_dir().path
eq_(os.path.exists(work_dir), True)
eq_(os.path.exists(work_dir), False)
def _local_file(self, resource):
url = Thread.currentThread().contextClassLoader.getResource(resource)
return self._session.local_file(File(url.toURI()))
def test_mkdirs_on_dir_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
eq_(os.path.exists(target.path + "/echo.sh"), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_file_copy(self):
target = self._session.work_dir_file("some/path/some.txt")
source = self._local_file("testfiles/echo.sh")
eq_(os.path.exists(target.path), False)
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
eq_(log_info[0], "Creating path %s" % target.parentFile.path)
eq_(log_info[1], "Copying %s to %s" % (source.path, target.path))
def test_mkdirs_on_turned_off_on_copy(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
success = False
try:
self._session.copy_to(source, target, mkdirs=False)
except:
success = True
eq_(success, True)
def test_delete_from_when_target_does_not_exit(self):
target = self._session.work_dir_file("some/path")
source = None
self._session.delete_from(source, target)
log_info = self._session.logger.output_lines
eq_(len(log_info), 1)
eq_(
log_info[0],
"Target [%s] does not exist. No deletion to be performed" %
target.path)
def test_delete_from_file_target(self):
target = self._session.work_dir_file("some/path/test.txt")
self._session.copy_text_to_file("xxx", target)
eq_(os.path.exists(target.path), True)
source = None
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 2)
self.assert_in_list(log_info, "Deleting [%s]" % target.path)
def test_delete_from_folder_target(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), False)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(
log_info,
"Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(log_info, "Deleting directory [%s]" % target.path)
def test_delete_from_with_external_resources(self):
target = self._session.work_dir_file("some/path")
source = self._local_file("testfiles")
self._session.copy_to(source, target)
eq_(os.path.exists(target.path), True)
external_resource = self._session.work_dir_file("some/path/ext.txt")
self._session.copy_text_to_file("xxx", external_resource)
eq_(os.path.exists(external_resource.path), True)
self._clear_logs()
self._session.delete_from(source, target)
eq_(os.path.exists(target.path), True)
log_info = self._session.logger.output_lines
eq_(len(log_info), 5)
self.assert_in_list(
log_info,
"Recursively deleting directory [%s/asubdir]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/echo.sh]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/multiline.txt]" % target.path)
self.assert_in_list(log_info,
"Deleting file [%s/singleline.txt]" % target.path)
self.assert_in_list(
log_info,
"Target directory [%s] is not shared, but still has content from an external source. Will not delete"
% target.path)
def test_copy_diff(self):
old = self._local_file("directorycompare/old")
new = self._local_file("directorycompare/new")
deployed_old = self._session.work_dir_file("olddeployed")
self._session.copy_to(old, deployed_old)
eq_(os.path.exists(deployed_old.path), True)
self._clear_logs()
diff = Diff.calculate_diff(old, new)
self._session.copy_diff(deployed_old.path, diff)
log_info = self._session.logger.output_lines
eq_(len(log_info), 17)
self.assert_in_list(log_info, "3 files to be removed.")
self.assert_in_list(log_info, "3 new files to be copied.")
self.assert_in_list(log_info, "2 modified files to be copied.")
self.assert_in_list(log_info, "Start removal of files...")
self.assert_in_list(
log_info, "Removing file %s/removefile.txt" % deployed_old.path)
self.assert_in_list(
log_info, "Removing directory %s/subdirremove" % deployed_old.path)
self.assert_in_list(
log_info,
"Removing file %s/subdirboth/removefile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Removal of files done.")
self.assert_in_list(log_info, "Start copying of new files...")
self.assert_in_list(
log_info, "Copying directory %s/subdirnew" % deployed_old.path)
self.assert_in_list(log_info,
"Copying file %s/newfile.txt" % deployed_old.path)
self.assert_in_list(
log_info,
"Copying file %s/subdirboth/newfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of new files done.")
self.assert_in_list(log_info, "Start copying of modified files...")
self.assert_in_list(
log_info, "Updating file %s/changedfile.txt" % deployed_old.path)
self.assert_in_list(
log_info,
"Updating file %s/subdirboth/changedfile.txt" % deployed_old.path)
self.assert_in_list(log_info, "Copying of modified files done.")
def test_execution_ctx_logging(self):
class ExecutionContext(object):
def __init__(self):
self.output_success = False
self.error_success = False
def logOutput(self, msg):
self.output_success = True
def logError(self, msg):
self.error_success = True
exec_ctx = ExecutionContext()
session = OverthereHostSession(self._linuxhost,
execution_context=exec_ctx)
session.logger.info("Check")
eq_(exec_ctx.output_success, True)
session.logger.error("Check")
eq_(exec_ctx.error_success, True)
class DarBuildServer(object):
def __init__(self, server):
self.retryCount = server['retryCount']
if self.retryCount == 0:
self.retryCount = 1
self.retryBaseInterval = 60
self.sshOpts = SshConnectionOptions(server['host'],
username=server['username'],
privateKeyFile=server['key_file'],
password=server['password'],
port=server['port'])
self.host = OverthereHost(self.sshOpts)
self.session = OverthereHostSession(self.host)
tries = 0
while tries < self.retryCount:
tries += 1
try:
self.zipBinary = server['pathToZipExecutable']
self.workingDirectory = server['workingDirectory']
self.create_directory(self.workingDirectory)
workDir = self.session.remote_file(self.workingDirectory)
self.session.get_conn().setWorkingDirectory(workDir)
break
except:
if tries > 3:
print sys.exc_info()[0]
print sys.exc_info()[1]
if tries > self.retryCount:
print sys.exc_info()[2].format_stack()
print "we were unable to setup a connection to server: %s after %i retries" % (
server['host'], tries)
sys.exit(2)
sleeptime = self.retryBaseInterval * tries
print "retrying to setup a connection to server %s in %i seconds" % (
server['host'], sleeptime)
time.sleep(sleeptime)
def __del__(self):
self.destroy_work_directory()
self.session.close_conn()
def closeConnection(self):
print "closing ssh connection"
self.session.close_conn()
@staticmethod
def createServer(server):
return DarBuildServer(server)
#
def init_dar(self, appName, appVersion):
workspace_root = self.create_dar_directory(appName, appVersion)
manifest_filename = "%s/deployit-manifest.xml" % workspace_root
self.write_dar_manifest(appName, appVersion, workspace_root)
def delete_workspace(self, appName, appVersion):
dirName = "%s/%s" % (appVersion, appName)
response = self.execute_command("/bin/rm -rf %s" % (dirName))
response = self.execute_command("/bin/rm -rf %s" % (appVersion))
def import_ear(self, appName, appVersion, deployable, url):
self.download_file_into_dar(appName, appVersion, deployable, str(url))
def create_dar_directory(self, appName, appVersion):
dirName = "%s/%s" % (appVersion, appName)
# check if the directory exists .. if it does we should go do something else.
# might want to do a better locking mechanism
if self.dir_exists(dirName):
print "unable to create dar directory: %s/%s/%s" % (
self.workingDirectory, appVersion, appName)
raise Exception('unable to create Dar Package Directory')
else:
self.create_directory(dirName)
return dirName
def download_file_into_dar(self, appName, appVersion, deployable, url):
#filename = url.split('/')[-1]
filename = os.path.basename(url)
outfile = "%s/%s/%s/%s" % (appVersion, appName, deployable, filename)
dirName = "%s/%s/%s" % (appVersion, appName, deployable)
if self.dir_exists(dirName):
print "output dir already exists: %s" % (dirName)
else:
self.create_directory(dirName)
self.execute_command(
"/usr/bin/curl --retry 5 --retry-delay 2 -k -L -o %s %s" %
(outfile, url))
def read_manifest(self, appName, appVersion):
file_name = "%s/%s/%s/deployit-manifest.xml" % (self.workingDirectory,
appVersion, appName)
return self.session.read_file(file_name, encoding="UTF-8")
def write_manifest(self, appName, appVersion, content):
file_name = "%s/%s/deployit-manifest.xml" % (appVersion, appName)
self.write_to_file(file_name, content)
def create_directory(self, dirName):
self.execute_command("/bin/mkdir -p %s" % (dirName))
def create_file(self, fileName, content=None):
if content:
self.write_to_file(fileName, str(content))
else:
self.execute_command("/bin/touch %s" % (fileName))
def write_to_file(self, fileName, content):
remoteFile = self.session.remote_file(
"%s/%s" % (self.workingDirectory, fileName))
self.session.copy_text_to_file(str(content), remoteFile)
def dir_exists(self, dirName):
command = "[ -d %s ]" % (dirName)
response = self.session.execute(command,
check_success=False,
suppress_streaming_output=False)
if response.rc == 0:
return True
else:
return False
def execute_command(self, command):
print "executing command: %s " % (command)
response = self.session.execute(command,
check_success=False,
suppress_streaming_output=False)
if response.rc != 0:
print response.stderr
print response.stdout
print "unable to execute command %s" % (command)
raise Exception('unable to execute command ')
else:
print "Response:", str(response.stdout)
print "Errors:", str(response.stderr)
# self.switch_working_directory()
return response
def write_dar_manifest(self, appName, appVersion, workspace_root):
filename = "./%s/deployit-manifest.xml" % (workspace_root)
file_content = self.basic_dar_manifest_template().substitute(
appVersion=appVersion, appName=appName)
self.create_file(filename, file_content)
def basic_dar_manifest_template(self):
xml_template = '<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n'
xml_template += ' <udm.DeploymentPackage version=\"$appVersion\" application=\"$appName\"> \n'
xml_template += ' <orchestrator /> \n'
xml_template += ' <deployables/> \n'
xml_template += ' </udm.DeploymentPackage> \n'
return Template(xml_template)
def package_dar(self, appName, appVersion):
command = "set -x \n"
command += "if [ -f %s_%s.dar ] \n"
command += " then rm -rf %s_%s.dar \n"
command += "fi \n"
command += "cd %s/%s \n" % (appVersion, appName)
command += "/usr/bin/zip -r %s/%s_%s.dar *" % (
self.workingDirectory, appName.replace(
'/', '_'), appVersion.replace('.', '_'))
self.execute_multiline_command(command)
def remove_dar(self, appName, appVersion):
command = "set -x \n"
command += "cd %s \n" % self.workingDirectory
command += "if [ -f %s_%s.dar ] \n" % (appName.replace(
'/', '_'), appVersion.replace('.', '_'))
command += " then rm -rf %s_%s.dar \n" % (appName.replace(
'/', '_'), appVersion.replace('.', '_'))
command += " fi \n"
self.execute_multiline_command(command)
def write_exec_script(self, commands, script):
self.session.upload_text_content_to_work_dir(self,
commands,
script,
executable=False)
def execute_multiline_command(self, command):
tmp_filename = self.filename_generator()
self.write_to_file(tmp_filename, command)
self.execute_command("chmod +x %s" % (tmp_filename))
self.execute_command("./%s" % (tmp_filename))
def filename_generator(self,
size=9,
chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def upload_dar_package(self, appName, appVersion, xldeployServer):
dar_file_name = "%s_%s.dar" % (appName.replace(
'/', '_'), appVersion.replace('.', '_'))
command = "/usr/bin/curl -k -u %s:%s -X POST -H \"content-type:multipart/form-data\" %s/package/upload/%s -F fileData=@./%s " % (
xldeployServer['username'], xldeployServer['password'],
xldeployServer['url'], dar_file_name, dar_file_name)
self.execute_command(command)