def manageService(self, services, service_action, pkg_dps, host):
try:
for service in services:
# check if package dependencies are already installed. If not, install them now
if service_action == 'start' or 'restart':
p = Packages()
pkgs_to_install = []
print("\n\n*** Checking dependencies...")
for pkg in pkg_dps:
if p.check_if_package_installed(pkg, host):
pass
else:
pkgs_to_install.append(pkg)
p.install_package(host, pkgs_to_install)
if service_action == 'stop':
pass
print("\n\n*** {}ing {} service on {}".format(service_action, service, host))
cmd = "service {} {}".format(service, service_action)
c = SshConnection()
output, error = c.run_command(cmd, host)
if error == []:
for o in output:
print(o.strip('\n'))
else:
raise Exception(error[0])
except Exception as e:
print(e)
def __init__(self, headers, team, arch):
super().__init__()
self._packages = Packages(team, arch)
# self.packages.get(self._setupModelData_) do this when ppa combo is selected
self.setHorizontalHeaderLabels(headers)
self.itemChanged.connect(TVModel.on_item_changed)
self._pool = multiprocessing.dummy.Pool(10)
def populate_repo(self, deploy_spec_name, deploy_spec, host, ssh_key,
local_root, conf_dir,
dry_run=True, verbose=False):
"""deploys all the packages in the deploy_spec to the given host
Args:
deploy_spec_name (string) - the name of the deploy spec
deploy_spec - the deploy_spec to extract the packages from
host - the host to push all the packages in the deploy_spec
ssh_key - the ssh key to use to connect to the push machine
local_root - the local root of the package repo
conf_dir - the current configuration directory location
"""
# check if the current prod/cluster/conf files are the SAME as the one
# in the package. if not, give the user two options.
# (1) continue as is
# (2) two update the releases.yaml AND generate a new cluster package
# with the specified package name AND exit, forcing the user
# to re-run the command
spec = self._create_spec(deploy_spec)
packages = self._packages_from_spec(spec)
if 'cluster' in packages:
sig = self._config_dir_sig(conf_dir, deploy_spec_name)
for cpkg in packages['cluster']:
lpath = os.path.join(local_root, 'cluster', cpkg, 'cluster/conf')
cpkg_sig = self._config_dir_sig(lpath, deploy_spec_name)
# check if the package signatures differ
if not sig == cpkg_sig:
s = raw_input(('Current cluster config files differ from the repo: %s.'
' Would you like to create a new cluster package?'
' Note: once you do, you will need to rerun the '
' command. ([Y]/N) ') % lpath)
if not s.strip().lower() == 'n':
os.chdir(subprocess.check_output(
'git rev-parse --show-toplevel', shell=True).strip())
subprocess.check_call(
'prod/update_packages.py --deployspec %s dummy %s' % (
deploy_spec_name, 'prod/config/cluster'),
shell=True)
raise Error(
'new cluster package generated please re-run the command!')
# create a new spec and packages with the new releases update
spec = self._create_spec(deploy_spec)
packages = self._packages_from_spec(spec)
# deploy the repo to prod
cluster_name = deploy_spec['cluster']
pkgs_obj = Packages(host,
user=self.clusters[cluster_name].get('user', ''), key=ssh_key,
dry_run=dry_run, verbose=verbose, compress=True)
pkgs_obj.get_versions() # preload versions of all packages on the host
for pname, pset in packages.iteritems():
for pver in pset:
pkgs_obj.push(self.local_root, pname, pver)
# activate the cluster config
if pname == 'cluster':
pkgs_obj.activate(pname, pver)
def generate_cluster_pkg(cls, release, name, release_fn):
"""Generate a cluster package with the latest config. The key
is to write the package name of the cluster package BEFORE
generating the package
Args:
release - the release read from releases.yaml. this struct is
UPDATED with the latest cluster package name
name - the release name
release_fn - the file_name to write the release
"""
pkg_prefix = Packages.get_valid_package_prefix()
pkg_name = Packages.generate_package_name(pkg_prefix, 'cluster')
release[name]['cluster'] = pkg_name
with open(release_fn, 'w') as f:
# writeback the releases config with the new release
f.write(yaml.safe_dump(release, default_flow_style=False))
# generate the package now that the new releases file is created
subprocess.check_call(
'flash --pkg_version_prefix=%s run prod/config/cluster' % pkg_prefix,
shell=True)
print 'generated cluster package %s and updated releases.yaml' % pkg_name
def clean_repo(self, hostgroup, ssh_key, dry_run=True, verbose=False):
required_pkg_versions = set()
for release, packages in sorted(self.releases.iteritems()):
for pkg_name, version in sorted(packages.iteritems()):
required_pkg_versions.add((pkg_name, version))
now = time.time()
for host in sorted(self.get_hosts([hostgroup])):
if verbose:
print "processing host", host
pkgs_obj = Packages(host, user='******', key=ssh_key,
dry_run=dry_run, verbose=verbose)
host_packages = pkgs_obj.get_packages()
for host_package in host_packages:
#print "processing package", host_package
host_package_versions = pkgs_obj.get_versions(host_package)
current = pkgs_obj.get_current(host_package)
for host_package_version in host_package_versions:
if current and host_package_version == current:
continue
if not (host_package, host_package_version) in required_pkg_versions:
if host_package == 'tmp':
version_ts = host_package_version.partition('__')[2] .partition('_')[0]
else:
version_ts = host_package_version.partition('_')[0]
try:
version_age = int((now - int(version_ts)) / 86400)
except (ValueError):
raise Error("%s: invalid version %s for packages %s" %(host, host_package_version, host_package))
if version_age > 15:
#print "delete", host, version_age, host_package, host_package_version
pkgs_obj.remove(host_package, host_package_version)
pass
def _find(self):
""" extracts and purge packages """
for repo in self._repos['list']:
if self._names and \
repo['name'] not in self._names:
continue
keywords = repo['keywords']
if self._keywords:
# overwrite repo definition
keywords = self._keywords
packages = Packages(repo['archs'], keywords)
packages.find(repo['testing'])
def _purged(path, name):
return not os.path.exists(os.path.join(path, name))
# XXX at this point packages is not longer Packages class
packages = [p for p in packages
if _purged(repo['updates'], p['filename'])]
repo['packages'] = packages
def analyse_packages(self): """ Generate information about all the installed packages. """ section = 'PACKAGES' self.payload['PACKAGES'] = dict() for cpv in Packages.get_installed_CPVs(): metadata = Metadata(cpv) package_info = dict() self.set_data(package_info, section, 'REPO', metadata.get_repo_name) self.set_data(package_info, section, 'SIZE', metadata.get_size) self.set_data(package_info, section, 'KEYWORD', metadata.get_keyword) self.set_data(package_info, section, 'BUILD_TIME', metadata.get_build_time) if self.any_one_is_enabled(section, USE_FLAG_TYPES): # TODO: make this lazier use_flags = metadata.get_use_flag_information() for key in USE_FLAG_TYPES: self.set_data(package_info, section, key, lambda: use_flags[key]) self.payload['PACKAGES'][cpv] = package_info
service_pkg_deps = parsed['action']['pkg_deps']
try:
c = SshConnection()
s = Service()
for host in hosts:
if commands:
for command in commands:
output, error = c.run_command(command, host)
if error != []:
raise Exception(error[0])
else:
print("\n*** Output of '{}' on '{}'".format(command, host))
for o in output:
print(o.strip('\n'))
elif install_pkgs:
p = Packages()
p.install_package(host, install_pkgs)
s = Service()
s.manageService(pkg_service_deps, 'restart', service_pkg_deps, host)
elif uninstall_pkgs:
p = Packages()
p.uninstall_package(host, uninstall_pkgs)
elif upgrade_pkgs:
p = Packages()
p.upgrade_package(host, upgrade_pkgs)
elif file_data and file_metadata:
f = File()
f.create_file(host, file_data, file_metadata)
elif not file_data and file_metadata:
f = File()
f.delete_file(host, file_metadata)
def main():
packages = Packages(output_package_info)
packages.check_for_updates()
if not updated:
print("Everything up-to-date.")
if sys.version_info < (3, 0):
reload(sys)
sys.setdefaultencoding("utf-8")
else:
# http://stackoverflow.com/questions/3828723/why-we-need-sys-setdefaultencodingutf-8-in-a-py-script
pass
# disable colors and simplify status symbols for Windows console
if sys.platform == "win32":
Colors.disable()
Symbols.simplify()
# packages updated
updated = Packages.create_counter()
def output_package_info(dist, available, status):
msg = None
if status == Packages.FAIL:
msg = u"{colors.fail}{symbols.fail}not found at PyPI{colors.end}".format(colors=Colors, symbols=Symbols)
elif status == Packages.UPDATED:
updated.append((dist, available))
msg = u"{colors.ok}{symbols.update}{colors.bold}{version}{colors.end}".format(colors=Colors, symbols=Symbols, version=available[0])
else:
if "-v" in sys.argv:
msg = u"{symbols.ok}up to date".format(colors=Colors, symbols=Symbols)
if msg:
result = u"{dist.project_name:30} {colors.bold}{dist.version:16}{colors.end} {msg}".format(colors=Colors, dist=dist, msg=msg)
if sys.version_info < (3, 0):
def do_packages(self, _args):
"""
Get information about packages installed on the device
"""
subconsole = Packages(self.session)
subconsole.cmdloop()
class TVModel(QStandardItemModel):
list_filled = pyqtSignal()
message = pyqtSignal(str)
exception = pyqtSignal('PyQt_PyObject')
def __init__(self, headers, team, arch):
super().__init__()
self._packages = Packages(team, arch)
# self.packages.get(self._setupModelData_) do this when ppa combo is selected
self.setHorizontalHeaderLabels(headers)
self.itemChanged.connect(TVModel.on_item_changed)
self._pool = multiprocessing.dummy.Pool(10)
@property
def packages(self):
return self._packages
def populate_pkg_list(self, ppa):
self.list_filled.emit()
self.removeRows(0, self.rowCount())
self._pool.apply_async(self._packages.populate_pkgs, (ppa, ),
callback=self.pkg_list_complete)
def pkg_list_complete(self, pkgs):
for pkg in pkgs:
pkg = TVItem(pkg)
if pkg.build_link in kfconf.cfg['installed']:
pkg.installed = Qt.Checked
else:
pkg.installed = Qt.Unchecked
self.appendRow(pkg.row)
self.list_filled.emit()
@staticmethod
def on_item_changed(item):
pkg = item.data(kfconf.TVITEM_ROLE)
if item.isCheckable():
if pkg.installed == Qt.Unchecked:
item.setCheckState(Qt.PartiallyChecked)
if item.checkState() == Qt.Unchecked:
if pkg.installed == Qt.PartiallyChecked:
kfconf.cfg['tobeinstalled'][pkg.ppa].pop(pkg.id)
kfconf.cfg.delete_ppa_if_empty('tobeinstalled', pkg.ppa)
if item.checkState() == Qt.PartiallyChecked:
if pkg.installed == Qt.Unchecked:
kfconf.cfg.add_item_to_section('tobeinstalled', pkg)
if pkg.installed == Qt.Checked:
kfconf.cfg.add_item_to_section('tobeuninstalled', pkg)
if item.checkState() == Qt.Checked:
item.setCheckState(Qt.Unchecked)
pkg.installed = item.checkState()
kfconf.cfg.filename = kfconf.config_dir + kfconf.CONFIG_FILE
kfconf.cfg.write()
def action_pkgs(self):
# uninstall packages that need uninstalling first.
# self._packages.uninstall_pkgs()
# install packages that need installing
self._packages.install_pkgs()
'dependencies', help='find dependencies for given packages')
dep_parser.add_argument('packages', nargs='*')
dep_parser.add_argument('-o',
'--or-dependency',
help='''allow choosing the or dependencies
of the package''',
action='store_true')
dep_parser.add_argument('-f',
'--facultative-package',
help='''pick also the falcultative
package''',
action='store_true')
args = parser.parse_args()
pkg_handler = Packages()
lastweek = time.time() - 7 * 24 * 3600
if args.update or not os.path.exists(PACKAGE_FILE_NAME) \
or os.stat(PACKAGE_FILE_NAME).st_mtime < lastweek:
sys.stdout.write('Downloading packages information... ')
download_packages_file()
print('Done')
sys.stdout.write('Beginning initializing... ')
pkg_handler.parse(read_packages_file())
print('Done\n')
if args.action == 'search':
for pkg in args.packages:
try:
def deploy_cmd(arg):
#print arg
p = Packages(arg['host'], user=arg['user'],
key=arg['key'], dry_run=arg['dry_run'],
verbose=arg['verbose'], compress=arg['compress'])
p.push(arg['local_root'],arg['package'], arg['version'])
def main():
packages = Packages(output_package_info)
packages.check_for_updates()
if not updated:
print("Everything up-to-date.")
from appearance import Colors, Symbols
if sys.version_info < (3, 0):
reload(sys)
sys.setdefaultencoding("utf-8")
else:
# http://stackoverflow.com/questions/3828723/why-we-need-sys-setdefaultencodingutf-8-in-a-py-script
pass
# disable colors and simplify status symbols for Windows console
if sys.platform == "win32":
Colors.disable()
Symbols.simplify()
# packages updated
updated = Packages.create_counter()
def output_package_info(dist, available, status):
msg = None
if status == Packages.FAIL:
msg = u"{colors.fail}{symbols.fail}not found at PyPI{colors.end}".format(
colors=Colors, symbols=Symbols)
elif status == Packages.UPDATED:
updated.append((dist, available))
msg = u"{colors.ok}{symbols.update}{colors.bold}{version}{colors.end}".format(
colors=Colors, symbols=Symbols, version=available[0])
else:
if "-v" in sys.argv:
msg = u"{symbols.ok}up to date".format(colors=Colors,
symbols=Symbols)
