def do_targz(self, args):
"""Create tar gz file creating directory too if not exists.
Usage:
targz source_dir output_filename
"""
arglist = args.split()
# Expect 2 argument
if len(arglist) != 2:
self.perror('targz requires 2 argument')
self.do_help('targz')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
source_file_or_dir = os.path.expanduser(arglist[0])
output_filename = os.path.expanduser(arglist[1])
# Create output directory if not exists
complete_path = pathlib.Path(output_filename)
path = pathlib.Path(os.sep.join(complete_path.parts[0:-1])[1:])
path.mkdir(parents=True, exist_ok=True)
# Create path for origin
origin_path = pathlib.Path(source_file_or_dir)
# Make tar gz
with tarfile.open(output_filename, "w:gz") as tar:
if origin_path.is_dir():
tar.add(source_file_or_dir,
arcname=os.path.basename(source_file_or_dir))
else:
files = self.list_files(source_file_or_dir)
for f in files:
tar.add(f)
def do_zstorevar(self, args):
"""Store all variables in init script.
Usage:
zstorevar
"""
arglist = args.split()
# Expect 0 argument
if len(arglist) != 0:
self.perror('zstorevar requires exactly 0 argument')
self.do_help('zstorevar')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Create empty startup script sfile
self.create_startup_file_if_not_exists()
# Open init script, delete all zset commands and insert the new zset commands at the beginning
lines = self.get_startup_script_as_list()
with open(self.STARTUP_SCRIPT, 'w') as f:
lines_without_set = [
line for line in lines if not line.lower().startswith('zset')
]
new_set = [
'zset {} {}\n'.format(key, value)
for key, value in self.store.items()
]
lines_with_new_set = new_set + lines_without_set
f.writelines(lines_with_new_set)
return
def do_untargz(self, args):
"""Create tar gz file creating directory too if not exists.
Usage:
untargz input_filename
untargz input_filename destination_dir
"""
arglist = args.split()
# Expect 1 or 2 argument
if len(arglist) > 2:
self.perror('untargz requires 1 or 2 argument')
self.do_help('untargz')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
input_file = os.path.expanduser(arglist[0])
destination = '.'
if len(arglist) == 2:
destination = os.path.expanduser(arglist[1])
# Create output directory if not exists
complete_path = pathlib.Path(destination)
path = pathlib.Path(os.sep.join(complete_path.parts[0:-1])[1:])
path.mkdir(parents=True, exist_ok=True)
# Decompress tar gz file
with tarfile.open(input_file, "r:gz") as tar:
tar.extractall(path=destination)
def do_dir(self, args, unknown):
"""List contents of current directory."""
# No arguments for this command
if unknown:
self.perror("dir does not take any positional arguments:")
self.do_help('dir')
self.last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Get the contents as a list
contents = os.listdir(self.cwd)
fmt = '{} '
if args.long:
fmt = '{}\n'
for f in contents:
self.stdout.write(fmt.format(f))
self.stdout.write('\n')
self.last_result = cmd2.CommandResult(data=contents)
def do_zinstall(self, args):
"""Install Z server.
Usage:
zinstall destination giturl version_name
"""
arglist = args.split()
# Expect 3 argument
if not arglist or len(arglist) != 3:
self.perror('zinstall requires exactly 3 argument:', traceback_war=False)
self.do_help('zinstall')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Recover arguments
destination = arglist[0]
giturl_raw = arglist[1]
version_name_arg = arglist[2]
# If exists, convert username and password to url encoded to build git url
giturl = self.get_git_url_encoded(giturl_raw)
# Check if version_name exists
if not self.version_exists(giturl_raw, version_name_arg):
self.perror('Version does not exist. Please check it.', traceback_war=False)
return
# If version name is "dev", it is neccesary to convert to the correct branch
version_name = self.convert_version_name(version_name_arg)
# Check if destination folder exists
if self.destination_exists(destination):
self.perror('Destination ifolder exists. Please check it and delete it', traceback_war=False)
return
# Clone repository to destination folder
self.poutput('Installing Z server')
self.clone_from_repository(giturl, version_name, destination)
# Create virtual environment and install requirements
venv_dir = os.path.abspath(destination + os.sep + '..' + os.sep + 'venv')
bin_dir = os.path.abspath(venv_dir + os.sep + 'bin')
os.system('python3 -m venv ' + venv_dir)
os.system(bin_dir + os.sep + 'pip install --no-cache-dir -r ' + destination + os.sep + 'requirements.txt')
# Set install path and version name variables and store them
self.do_zset('{} {} store'.format(VARIABLE_INSTALL_PATH, destination))
self.do_zset('{} {} store'.format(VARIABLE_VERSION_NAME, version_name_arg))
self.do_zset('{} {} store'.format(VARIABLE_GIT_URL, giturl_raw))
self.do_zset('{} {} store'.format(VARIABLE_VENV_BIN_PATH, bin_dir))
self.poutput('Z server installed')
def do_zupgrade(self, args):
"""Upgrade application.
Usage:
zupgrade
zupgrade version_name
"""
# Recover install path. git url and version name
install_path = self.store.get(VARIABLE_INSTALL_PATH, None)
older_version_name = self.store.get(VARIABLE_VERSION_NAME, None)
giturl = self.store.get(VARIABLE_GIT_URL, None)
arglist = args.split()
# Expect 0 or 1 argument
if len(arglist) > 1:
self.perror('zupgrade requires 0 or 1 argument:', traceback_war=False)
self.do_help('zupgrade')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
if len(arglist) == 0:
# Recover last version from git:
version_name = self.get_last_tag(giturl)
version_name_arg = version_name
elif len(arglist) == 1:
version_name_arg = arglist[0]
version_name = version_name_arg
if version_name_arg.lower() == DEVELOPER_BRANCH[0]:
version_name = DEVELOPER_BRANCH[1]
if install_path and older_version_name and giturl:
# Stop server
self.do_zstop(None)
# Compress actual deployment for backup generating compressed file to backup folder
backup_folder = install_path + os.sep + '..' + os.sep + 'backup' + os.sep
filename = '{}-{}.tar.gz'.format(self.get_timestamp(), older_version_name)
compressed_file = backup_folder + filename
self.do_targz(install_path + ' ' + compressed_file)
# Delete installation
shutil.rmtree(install_path)
# Install the new version
self.poutput('Upgrading Z server')
self.do_zinstall(install_path + ' ' + giturl + ' ' + version_name_arg)
else:
self.poutput('No installation folder found')
def do_cd(self, arglist):
"""Change directory.
Usage:
cd <new_dir>
"""
# Expect 1 argument, the directory to change to
if not arglist or len(arglist) != 1:
self.perror("cd requires exactly 1 argument")
self.do_help('cd')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Convert relative paths to absolute paths
path = os.path.abspath(os.path.expanduser(arglist[0]))
# Make sure the directory exists, is a directory, and we have read access
out = ''
err = None
data = None
if not os.path.isdir(path):
err = '{!r} is not a directory'.format(path)
elif not os.access(path, os.R_OK):
err = 'You do not have read access to {!r}'.format(path)
else:
try:
os.chdir(path)
except Exception as ex:
err = '{}'.format(ex)
else:
out = 'Successfully changed directory to {!r}\n'.format(path)
self.stdout.write(out)
data = path
if err:
self.perror(err)
self._last_result = cmd2.CommandResult(out, err, data)
def do_sendgmail(self, args):
"""
Send mail through gmail
:param args: json file with mail data
:return:
"""
arglist = args.split()
# Expect 1 argument
if not arglist or len(arglist) != 1:
self.perror('sendmail requires one argument')
self.do_help('sendmail')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Recover arguments
file = arglist[0]
with open(file) as f:
data = json.load(f)
# Connecto to gmail
yag = yagmail.SMTP(user=data.get('from'),
password=data.get('password'))
# Send mails
for i in range(data.get('sendingNumber')):
# Replace vars
subject = data.get('subject')
body = data.get('body')
var_datetime = dt_string = datetime.datetime.now().strftime(
"%d/%m/%Y %H:%M:%S")
subject = subject.replace('{DATETIME}', var_datetime)
body = body.replace('{DATETIME}', var_datetime)
yag.send(
to=data.get('to'),
cc=data.get('cc'),
bcc=data.get('bcc'),
subject=subject,
contents=body,
attachments=data.get('attachments'),
)
self.poutput('Mail {} sent'.format(i + 1))
time.sleep(data.get('secondsBetweenSending'))
def do_zset(self, args):
"""Set variable in Z server.
Usage:
zset varname value
zset varname value store -> With the keyword store it saves the variable in startup script
"""
arglist = args.split()
# Expect 2 argument
if not arglist or len(arglist) < 2 or len(arglist) > 3:
self.perror('zset requires 2 or 3 argument')
self.do_help('zset')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Recover arguments
key = arglist[0]
value = arglist[1]
save = False
if len(arglist) == 3:
save = True if arglist[2].lower() == 'store' else False
# Update dictionary
self.store.update({key: value})
# Update startup script is save is set
if save:
self.create_startup_file_if_not_exists()
lines = self.get_startup_script_as_list()
# Open startup script and add at the beginning this variable
# Be careful because it is possible that already exists the variable in startup script: delete before write
with open(self.STARTUP_SCRIPT, 'w') as f:
new_command = ['zset {} {}\n'.format(key, value)]
# Delete variable from lines if it exists
lines_modified = [
line for line in lines if
not line.lower().startswith('zset {}'.format(key.lower()))
]
lines_with_new_command = new_command + lines_modified
f.writelines(lines_with_new_command)
self.poutput('Variable stored {} = {}'.format(key, value))
def do_zdel(self, args):
"""Delete variable value in Z server.
Usage:
zdel varname
"""
arglist = args.split()
# Expect 1 argument
if not arglist or len(arglist) != 1:
self.perror('zdel requires exactly 1 argument')
self.do_help('zdel')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
key = arglist[0]
del self.store[key]
return
def do_zgetlastversion(self, args):
"""Recover last version from Z server.
Usage:
getlastversion
"""
arglist = args.split()
# Expect 0 argument
if len(arglist) != 0:
self.perror('zgetlastversion requires exactly 0 argument:', traceback_war=False)
self.do_help('zgetlastversion')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
actual_version_name = self.store.get(VARIABLE_VERSION_NAME, None)
giturl = self.store.get(VARIABLE_GIT_URL, None)
self.poutput('Actual version: {}'.format(actual_version_name))
self.poutput('Last version: {}'.format(self.get_last_tag(giturl)))
def do_sendget(self, args):
"""
Send get request
Usage:
sendget url csv_file
"""
arglist = args.split()
# Expect 2 argument
if len(arglist) != 2:
self.perror('sendget requires exactly 2 argument')
self.do_help('sendget')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Recover arguments
url = arglist[0]
file = arglist[1]
with codecs.open(file, 'rU', 'utf-16') as f:
csv_reader = csv.reader(f, delimiter=',')
# File number
file_number = 0
for row in csv_reader:
# Replace url with data from csv
i = 0
url_transformed = url
for data in row:
url_transformed = url_transformed.replace(
'${}'.format(i), data)
i = i + 1
# Send get request
r = requests.get(url_transformed)
self.poutput('GET request {} {} sent with output {} {}'.format(
file_number, url_transformed, r.status_code, r.text))
file_number = file_number + 1
def do_zget(self, args):
"""Get variable value in Z server. Return 'NOT SET string if not exists.
Usage:
zget
zget varname
"""
arglist = args.split()
# Expect 0 or 1 argument
if len(arglist) > 1:
self.perror('zget requires 0 or 1 argument')
self.do_help('zget')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
if len(arglist) == 0:
for (key, value) in self.store.items():
self.poutput('{}: {}'.format(key, value))
elif len(arglist) == 1:
key = arglist[0]
value = self.store.get(key, 'NOT SET')
self.poutput(value)
def do_zremoteupgrade(self, args):
"""Upgrade Z server in remote server.
Usage:
zremoteupgrade sshconnection
zremoteupgrade sshconnection version_name
sshconnection --> ssh://username:password@server:port
"""
arglist = args.split()
# Expect 0 or 1 argument
if len(arglist) < 1 or len(arglist) > 2:
self.perror('zremoteupgrade requires 1 or 2 argument:', traceback_war=False)
self.do_help('zremoteupgrade')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Recover sshconnection
ssh_connection = arglist[0]
# Recover install path. git url and version name
remote_variables = self.get_variables_from_startup_script(ssh_connection)
install_path = remote_variables.get(VARIABLE_INSTALL_PATH, None)
older_version_name = remote_variables.get(VARIABLE_VERSION_NAME, None)
giturl = remote_variables.get(VARIABLE_GIT_URL, None)
if len(arglist) == 2:
version_name_arg = arglist[1]
else:
version_name_arg = self.get_last_tag(giturl)
# Convert version_name is dev is selected
version_name = version_name_arg
if version_name_arg.lower() == DEVELOPER_BRANCH[0]:
version_name = DEVELOPER_BRANCH[1]
if install_path and older_version_name and giturl:
# Do some actions in remote server: stop, backup, delete and install
# ssh_connection --> ssh://username:password@server:port
url = urllib.parse.urlparse(ssh_connection)
with spurplus.connect_with_retries(
retries=5,
hostname=url.hostname,
username=url.username,
password=url.password if url.password else None,
port=url.port if url.port else None
) as shell:
# Stop server
zctl_path = install_path + os.sep + ZCTL_NAME
command = [zctl_path] + ['zstop'] + ['quit']
shell.run(command)
# Compress actual deployment for backup generating compressed file to backup folder
backup_folder = install_path + os.sep + '..' + os.sep + 'backup' + os.sep
filename = '{}-{}.tar.gz'.format(self.get_timestamp(), older_version_name)
compressed_file = backup_folder + filename
# Execute targz command
command = [zctl_path] + ['targz {} {}'.format(install_path, compressed_file)] + ['quit']
shell.run(command)
# Delete installation
command = ['rm'] + ['-rf'] + [install_path]
shell.run(command)
# Install the new version
self.poutput('Upgrading Z server')
self.do_zremoteinstall(install_path + ' ' + giturl + ' ' + version_name_arg + ' ' + ssh_connection)
else:
self.poutput('No installation folder found')
def do_zremoteinstall(self, args):
"""Install Z server in remote server.
Usage:
zremoteinstall destination giturl version_name sshconnection
sshconnection --> ssh://username:password@server:port
"""
arglist = args.split()
# Expect 3 argument
if not arglist or len(arglist) != 4:
self.perror('zremoteinstall requires exactly 4 argument:', traceback_war=False)
self.do_help('zremoteinstall')
self._last_result = cmd2.CommandResult('', 'Bad arguments')
return
# Recover arguments
destination = arglist[0]
giturl_raw = arglist[1]
version_name_arg = arglist[2]
ssh_connection = arglist[3]
# If exists, convert username and password to url encoded to build git url
giturl = self.get_git_url_encoded(giturl_raw)
# Check if version_name exists
if not self.version_exists(giturl_raw, version_name_arg):
self.perror('Version does not exist. Please check it.', traceback_war=False)
return
# If version name is "dev", it is neccesary to convert to the correct branch
version_name = self.convert_version_name(version_name_arg)
# Check if destination folder exists
if self.destination_exists(destination, ssh_connection):
self.perror('Destination ifolder exists. Please check it and delete it', traceback_war=False)
return
# Define and delete if exists destination_tmp and venv_dir_tmp
destination_tmp = tempfile.gettempdir() + os.sep + 'install'
venv_dir_tmp = os.path.abspath(destination_tmp + os.sep + '..' + os.sep + 'venv')
if self.destination_exists(destination_tmp):
shutil.rmtree(destination_tmp)
if self.destination_exists(venv_dir_tmp):
shutil.rmtree(venv_dir_tmp)
# Clone repository to temporal destination folder
self.poutput('Installing Z server')
self.clone_from_repository(giturl, version_name, destination_tmp)
# Create virtual environment and install requirements
bin_dir_tmp = os.path.abspath(venv_dir_tmp + os.sep + 'bin')
os.system('python3 -m venv ' + venv_dir_tmp)
os.system(bin_dir_tmp + os.sep + 'pip install --no-cache-dir -r ' + destination_tmp + os.sep +
'requirements.txt')
# Define venv_dir. Define python binary folder
venv_dir = os.path.abspath(destination + os.sep + '..' + os.sep + 'venv')
bin_dir = os.path.abspath(venv_dir + os.sep + 'bin')
# Replace all ocurrences of venv_dir_tmp for the new venv folder in destination
self.find_replace(bin_dir_tmp, venv_dir_tmp, venv_dir)
# Replace shebang (#!) for venv python in destination_tmp files
old_shebang = '#!/usr/bin/python3'
new_shebang = '#!{}'.format(bin_dir) + os.sep + 'python'
zctl_path_tmp = destination_tmp + os.sep + ZCTL_NAME
self.find_replace(zctl_path_tmp, old_shebang, new_shebang, first_occurrence=True)
# Check operations in remote server
url = urllib.parse.urlparse(ssh_connection)
with spurplus.connect_with_retries(
retries=5,
hostname=url.hostname,
username=url.username,
password=url.password if url.password else None,
port=url.port if url.port else None
) as shell:
# Delete virtual environment if exist
if shell.exists(venv_dir):
shell.remove(venv_dir, recursive=True)
# Copy temporal virtual environment and temporal installation folder to remote server
shell.mkdir(remote_path=destination, parents=True, exist_ok=True)
shell.mkdir(remote_path=venv_dir, parents=True, exist_ok=True)
self.poutput('Copying Z server')
shell.sync_to_remote(
local_path=destination_tmp,
remote_path=destination,
delete=spurplus.Delete.BEFORE,
preserve_permissions=True
)
self.poutput('Copying virtual environment')
shell.sync_to_remote(
local_path=venv_dir_tmp,
remote_path=venv_dir,
delete=spurplus.Delete.BEFORE,
preserve_permissions=True
)
# Set install path and version name variables and store them
# Execute these commands in remote server
zctl_path = destination + os.sep + ZCTL_NAME
command_1 = 'zset {} {} store'.format(VARIABLE_INSTALL_PATH, destination)
command_2 = 'zset {} {} store'.format(VARIABLE_VERSION_NAME, version_name_arg)
command_3 = 'zset {} {} store'.format(VARIABLE_GIT_URL, giturl_raw)
command_4 = 'zset {} {} store'.format(VARIABLE_VENV_BIN_PATH, bin_dir)
command_5 = 'quit'
zcommand = '{} {} {} {} {}'.format(command_1, command_2, command_3, command_4, command_5)
chmod_command = ['chmod'] + ['+x'] + [zctl_path]
command = [zctl_path] + [command_1] + [command_2] + [command_3] + [command_4] + [command_5]
shell.run(chmod_command)
shell.run(command)
self.poutput('Z server installed')
