def run_backup():
""" git clone/pull + rsync + git commit/push. Since Jenkins integration ,
this command doesn't up the .deb into reprepro. Jenkins will do this."""
params = request.json
remotehost = params['host']
appName = params['app']
branch = 'master'
language = 'none'
command = 'backup'
msg_stdout = ''
msg_return = ''
msg_git_push = ''
''' Clone/pull from git + Rsync '''
msg_stdout = action.build_from_git(appName, branch, language, command,
remotehost)
l.info(msg_stdout['Stdout'])
''' Add new changes, commit and push '''
if not msg_stdout['Error']:
msg_git_push = action.git_push(appName, branch)
concatenate_git_output = '\n\n %s' % msg_git_push
msg_stdout['Stdout'] += concatenate_git_output
''' Jenkins will make the new package from source '''
if msg_stdout['Stdout']:
msg_return = msg_stdout['Stdout']
else:
msg_return = 'Unknown error: please contact with Buanarepo Admin or take a look to Buanarepo-Server logs.'
l.error(msg_return)
return msg_return
def run_git():
""" If a git repository doesn't exists into buanarepo's build path (ej.
/srv/buanarepo-build), This repository will be cloned. Later, a new .deb pkg will be created """
params = request.json
appName = params['app']
branch = params['branch']
language = params['language']
command = 'git'
msg_return = ''
msg_stdout = ''
# Clone or pull Git repository
msg_stdout = action.build_from_git(appName,
branch,
language,
command,
remotehost=None)
if msg_stdout['Stdout']:
msg_return = msg_stdout['Stdout']
else:
msg_return = 'Unknown error: please contact with Buanarepo Admin or take a look to Buanarepo-Server logs.'
l.error(msg_return)
return msg_return
def listPackage(self):
l_packages = []
pkg_file = None
basic_auth = (API_USER, USER_PASS)
try:
request = get(URL_PKG, pkcs12_filename=CERT_P12, pkcs12_password=CERT_P12_PASS,\
verify=CA_CERT, auth=basic_auth)
#request = urllib2.Request(URL_PKG)
#response = urllib2.urlopen(request, context=ssl._create_unverified_context())
pkg_file = request.text
except Exception as e:
l.error(e)
result_packages = e
if not pkg_file:
l_packages = result_packages
else:
result_packages = re.findall('Package.*', pkg_file)
for package in result_packages:
pkg = package.split()
l_packages.append(pkg[1])
return l_packages
def find_ldap_user(attrlist, dn):
'''
从ldap批量取用户信息
:param attrlist: 需要的字段，如 ['sAMAccountName', 'name', 'userAccountControl', 'msExchHomeServerName']
:param dn: DN条件，查询的范围
:return: account_list普通账号的信息
'''
import ldap.controls
# 分页查询
page_control = ldap.controls.SimplePagedResultsControl(True,
size=1000,
cookie='')
# 最后生成的用户列表
account_list = []
try:
con = con_ldap()
while 1:
msgid = con.search_ext(dn,
ldap.SCOPE_SUBTREE,
attrlist=attrlist,
serverctrls=[page_control])
a, res_data, b, srv_ctrls = con.result3(msgid)
for info in res_data:
# TODO ：此处可去除这两种条件限制，在后面过滤
# msExchHomeServerName 表明用户具有exchange server邮箱，而不是其他类型的AD账号
if 'userAccountControl' in str(info[1]) and 'Exchange' in str(
info[1]):
if PY2:
# userAccountControl记录AD账号的属性，该属性标志是累积性的，32不需要密码，512是普通账号66048=65536+512,514是禁用账号，
if info[1]['userAccountControl'][0] in [
'512', '544', '66048', '66080', '262656',
'262688', '328192', '328224'
]:
account_list.append(info)
elif PY3:
if info[1]['userAccountControl'][0] in [
b'512', b'544', b'66048', b'66080', b'262656',
b'262688', b'328192', b'328224'
]:
account_list.append(info)
cookie = srv_ctrls[0].cookie
if cookie:
page_control.cookie = cookie
else:
break
except Exception as e:
logger.error(e)
return account_list
def build_from_git(self, appName, branch, language, command, remotehost):
''' call clone_from_git and build package from source cloned '''
msg_stdout_git = ''
msg_build_pkg = ''
msg_stdout = self.clone_from_git(appName, language)
if not msg_stdout['Error']:
if command == 'backup':
appLoader = AppLoader(MOD_PATH)
""" It's like 'from lib.Kafka import Kafka' and 'loadClass = Kafka()' """
try:
l.info('Loading %s class for sync...' % appName)
loadClass = appLoader.get_instance(appName)
except ImportError as e:
stderr = str(e)
msg_stdout['Stdout'] = 'Error: Application "%s" not found'\
% appName
msg_stdout['Error'] = True
l.error(stderr)
else:
try:
msg_stdout['Stdout'] = loadClass.backup(remotehost)
l.info(msg_stdout['Stdout'])
except Exception as e:
msg_exception = str(e)
l.error(msg_exception)
msg_stdout['Stdout'] = msg_exception
msg_stdout['Error'] = True
else:
try:
msg_build_pkg = self.build_pkg(appName, command, branch,language)
l.info(msg_build_pkg['Stdout'])
msg_stdout['Stdout'] = msg_build_pkg['Stdout']
msg_stdout['Error'] = msg_build_pkg['Error']
except Exception as e:
msg_stdout['Stdout'] = str(e)
msg_stdout['Error'] = True
return msg_stdout
def verify_password(request_username, request_password):
access = ''
if request_username in users:
m = md5.new()
m.update(request_password)
md5_pass = m.hexdigest()
if md5_pass == users.get(request_username):
l.info('User/Passwd Authentication is OK!')
access = True
else:
l.error(
'Wrong User or Password. Please edit password and try again.')
access = False
else:
l.error('User not found!')
return access
def run_mpkg():
""" Create package from source whitout pull/clon the code."""
params = request.json
appName = params['app']
branch = params['branch']
language = params['language']
default_command = 'mpkg'
msg_return = {}
# Make the package
msg_return = action.make_pkg(appName, branch, language)
if msg_return['Stdout']:
msg_return = msg_return['Stdout']
else:
msg_return = 'Unknown error: please contact with Buanarepo Admin or take a look to Buanarepo-Server logs.'
l.error(msg_return)
return msg_return
def useSubprocess(self, cmd):
msg_return = {'Stdout': '', 'Error': False}
stdout = ''
stderr = ''
try:
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
except Exception as e:
l.error(e)
msg_return['Stdout'] = e
msg_return['Error'] = True
if not stdout:
msg_return['Stdout'] = stderr
msg_return['Error'] = True
else:
msg_return['Stdout'] = stdout
return msg_return
def git_push(self, appName, branch):
''' Commit the changes '''
appName = str(appName).lower()
path_local_repo = '%s/%s' % (BASE_BUILD, appName)
remote_url = 'https://%s:[email protected]/%s/' % (TOKEN, COMPANY)
remote_repo = remote_url + appName
msg_stdout = ''
try:
repo = Repo(path_local_repo)
except InvalidGitRepositoryError as e:
msg_stdout = str(e)
l.error(msg_stdout)
except NoSuchPathError as e:
msg_stdout = str(e)
l.error(msg_stdout)
else:
origin = repo.remote(name='origin')
""" If in this point is present the next API error "list out of range", the problem is git control file"""
try:
l.info('Commit the new changes at %s' % path_local_repo)
git = repo.git
git.add ('.')
repo.index.commit('New %s configuration' % appName)
origin.push()
msg_stdout = 'The %s latest version was pushed!' % appName
except Exception as e:
msg_stdout = str(e)
l.error(msg_stdout)
return msg_stdout
def clone_from_git(self, appName, language):
appName = str(appName).lower()
path_local_repo = '%s/%s' % (BASE_BUILD, appName)
remote_url = 'https://%s:[email protected]/%s/'\
% (TOKEN, COMPANY)
remote_repo = remote_url + appName
msg_stdout = {
'Stdout' : 'Error: %s isn\'t a valid local repository'\
% path_local_repo, 'Error' : True
}
try:
repo = Repo(path_local_repo)
msg_stdout['Error'] = False
except InvalidGitRepositoryError as e:
l.error(msg_stdout['Stdout'])
msg_stdout['Error'] = True
except NoSuchPathError:
try:
l.info('Clonning %s repository' % remote_repo)
msg_stdout['Stdout'] = Repo.clone_from(remote_repo,\
path_local_repo, depth=1)
msg_stdout['Error'] = False
except GitCommandError as gitException:
if re.findall(r'Repository not found.', gitException.stderr):
msg_stdout['Stdout'] = gitException.stderr
msg_stdout['Error'] = True
l.error(msg_stdout['Stdout'])
else:
l.info('Repository %s already exists, doing pull...' % path_local_repo)
origin = repo.remote(name='origin')
""" If in this point is present the next API error "list out of range", the problem is git control file"""
try:
msg_stdout['Stdout'] = origin.pull()
msg_stdout['Error'] = False
l.info(msg_stdout['Stdout'])
except Exception as e:
msg_stdout['Stdout'] = str(e)
msg_stdout['Error'] = True
l.error(msg_stdout['Stdout'])
return msg_stdout
def make_sync(self, appName, branch):
msg_return = ''
appName = str(appName).lower()
cmd_ls = CMD_LS % (DEBS_BASE, appName)
search_package = self.useSubprocess(cmd_ls)
if not search_package['Error']:
package = search_package['Stdout']
for component in COMPONENTS:
cmd_remove = CMD_REMOVE % (UBUNTU_BASE, branch, component,\
appName)
remove_package = self.useSubprocess(cmd_remove)
if not remove_package['Error']:
cmd_upload = CMD_UPLOAD % (UBUNTU_BASE, branch,\
component, DEBS_BASE, package)
l.info(remove_package['Stdout'])
upload_package = self.useSubprocess(cmd_upload)
if upload_package['Error']:
l.error(upload_package['Stdout'])
raise Exception(msg_return)
else:
msg_return += upload_package['Stdout']
l.info(upload_package['Stdout'])
else:
msg_return = remove_package['Stdout']
l.error(remove_package['Stdout'])
raise Exception(msg_return)
else:
msg_return = search_package['Stdout']
l.error(search_package['Stdout'])
raise Exception(msg_return)
return msg_return
def orchestrator(self, host, services, l_msg):
RemoteDeployUser = REMOTE_DEPLOY_USER
UpdateCommand = UPDATE_CMD
InstallCommand = INSTALL_CMD
msg_action = []
proc = os.getpid()
try:
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(host, username=RemoteDeployUser, timeout=30.0)
upd_cmd = UpdateCommand
stdin, stdout, stderr = client.exec_command(upd_cmd)
except SSHException as error:
cmd_stdout = self.create_msg(host, error[0], 'host')
l.error(cmd_stdout)
msg_action.append(cmd_stdout)
except AuthenticationException as error:
cmd_stdout = self.create_msg(host, error[0], 'host')
l.error(cmd_stdout)
msg_action.append(cmd_stdout)
except BadHostKeyException as error:
cmd_stdout = self.create_msg(host, error[0], 'host')
l.error(cmd_stdout)
msg_action.append(cmd_stdout)
except Exception as error:
cmd_stdout = self.create_msg(host, 'Destination Host Unreachable',\
'host')
l.error(cmd_stdout)
msg_action.append(cmd_stdout)
else:
for service in services:
ins_cmd = '%s %s' % (InstallCommand, service)
l.info(ins_cmd)
try:
stdin, stdout, stderr = client.exec_command(ins_cmd)
exit_status = stdout.channel.recv_exit_status()
if exit_status == 0:
cmd_stdout = self.create_msg(service, 'ok', 'service')
l.info(cmd_stdout)
msg_action.append(cmd_stdout)
else:
cmd_stdout = self.create_msg(service, 'error', 'service')
l.error(cmd_stdout)
msg_action.append(cmd_stdout)
except SSHException as error:
cmd_stdout = self.create_msg(service, error[0], 'service')
l.error(cmd_stdout)
msg_action.append(cmd_stdout)
"""When the command finishes executing, the channel will be closed and can’t be reused. You must open a new channel if you wish to execute another command."""
client.close()
msg_action = {'Host': host, 'Install': msg_action}
l_msg.append(msg_action)
def build_pkg(self, appName, command, branch, language=None):
""" Builds packages from local dirs or git repositories. """
multiPackagePath = MULTI_PACKAGE_PATH
path_local_repo = '%s/%s' % (BASE_BUILD, appName)
local_jenkins_repo = '%s/%s' % (BASE_JENKINS_BUILD, appName)
appName = str(appName).lower()
msg_return = {'Stdout': '', 'Error': False}
msg_exception = ''
if command == 'git':
base_path = '%s/%s' % (path_local_repo, appName)
appDpkgSource = '%s/%s' % (appName, appName)
# If multi-package folder exists, then process sub-packages in it:
sub_package_path = '%s/%s' % (path_local_repo, multiPackagePath)
elif command == 'mpkg':
base_path = '%s/%s' % (local_jenkins_repo, appName)
appDpkgSource = '%s/%s' % (appName, appName)
sub_package_path = '%s/%s' % (local_jenkins_repo, multiPackagePath)
elif command == 'backup':
base_path = '%s' % (path_local_repo)
appDpkgSource = appName
l.info('Checks multi-package path: %s' % sub_package_path)
if os.path.isdir(sub_package_path):
l.info('Running multi-package case...')
try:
appBaseSource = '%s/%s' % (appName, multiPackagePath)
msg_return = self.build_several_pkgs(sub_package_path,\
appBaseSource, command, branch, language)
if msg_return['Error']:
l.error(msg_return['Stdout'])
else:
l.info(msg_return['Stdout'])
except Exception as msg_exception:
l.error(msg_exception)
msg_return['Stdout'] = str(msg_exception)
msg_return['Error'] = True
else:
l.info('No multi-package case detected.')
l.info('Building %s...' % appName)
# Single package case
try:
control_file = '%s/DEBIAN/control' % base_path
appVersion = self.get_app_version(control_file)
pkgName = self.get_app_name(control_file)
msg_return = self.dpkg(appName, appDpkgSource, appVersion,\
pkgName, command, branch, language)
if msg_return['Error']:
l.error(msg_return['Stdout'])
else:
l.info(msg_return['Stdout'])
except Exception as msg_exception:
l.error(msg_exception)
msg_return['Stdout'] = str(msg_exception)
msg_return['Error'] = True
return msg_return