def architecture_can_be_build(self: object) -> None: build_architectures = sh.dpkg("--print-foreign-architectures", _tty_out=False).strip().split() build_architectures.append( sh.dpkg("--print-architecture", _tty_out=False).strip()) if self.host_architecture not in build_architectures: raise ValueError("Host architecture {} cannot be build.".format( self.host_architecture))
def _cmp(self, oper, other): if not isinstance(other, V): raise TypeError("Stop sucking.") if oper not in self._opers: raise ValueError("Blow it out 'yer ass") try: dpkg("--compare-versions", self.version_string, oper, other.version_string) return True except sh.ErrorReturnCode_1: return False
def execute(self): try: sh.dpkg("-l", self.package) except sh.ErrorReturnCode_1: try: with Authentication(): sh.apt_get("install", "-y", self.package) except sh.ErrorReturnCode as err: err_message = "\n\t" + err.stderr.replace("\n", "\n\t") logging.error( "Error with `apt-get install %s`: %s", self.package, err_message ) return False return True
def main(): dpkg_output = dpkg('-l', 'linux-image-[0-9]*', 'linux-headers-[0-9]*', _env={'COLUMNS': '200'}) uname_output = uname('-r').strip() # like 4.4.0-79-generic kernel_version = uname_output.rsplit('-', 1)[0] print('kernel version:', kernel_version) removed_packages = filter_packages(dpkg_output, kernel_version) print('\n'.join(removed_packages))
def package_installed(package): from sh import dpkg, ErrorReturnCode print(package) try: response = dpkg("-s", package) m = re.search("Version:\s(.:)?(\S*)\\n", str(response)) return m.groups()[1].split(".") except ErrorReturnCode as e: return False
def check_if_package_is_installed(self, package): """checks if a package is installed :param string package: package name to check :rtype: `bool` representing whether package is installed or not """ lgr.debug('Checking if {0} is installed'.format(package)) o = sh.dpkg('-s', package) if not o.exit_code: lgr.debug('{0} is installed'.format(package)) return True else: lgr.error('{0} is not installed'.format(package)) return False
def is_package_installed(self) -> bool: if self.package_manager == PackageManager.PACMAN: installed_query_command = pacman("-Qi", self.package, _ok_code=[0, 1], _env=configfinder.config_settings. newenv) # type:sh.RunningCommand elif self.package_manager == PackageManager.APT: installed_query_command = dpkg( "-s", self.package, _ok_code=[0, 1], _env=configfinder.config_settings.newenv) else: raise ValueError("No supported package manager found!") if installed_query_command.exit_code == 0: print("Package {0} is already installed!".format(self.package)) return True elif installed_query_command.exit_code == 1: return False
def read_settings(): parser = argparse.ArgumentParser( description= 'Create a debian package out of a pipenv enabled python project') parser.add_argument( '--script', action='append', help='Script for the tool. Can be read from the setup.py file') parser.add_argument( '--name', help='Name of the tool. Can be read from the setup.py file') parser.add_argument( '--version', help='Version of the tool. Can be read from setup.py file') parser.add_argument( '--project', help= 'Root folder for project. Defaults to the folder containing the Pipfile' ) parser.add_argument('--python-version', default='3.6.5', help='Python version to install') parser.add_argument('pipfile', help='Path to the Pipfile for the project') parser.add_argument( 'setup', nargs='?', help= 'Optional path to the setup.py file. Will read settings from this file' ) args = parser.parse_args() pipfile = abspath(args.pipfile) project = args.project or dirname(pipfile) name = None version = None scripts = None if args.setup: with open(args.setup, 'r') as handle: name, version, scripts = read_setup_details(handle.read()) if args.name: name = args.name if args.version: version = args.version if args.script: scripts = args.script if not name: name = basename(project) if not version: version = '0.0.0' if not scripts: scripts = [] architecture = dpkg('--print-architecture') return Settings( project=project, name=name, version=version, scripts=scripts, pipfile=pipfile, python_version=args.python_version, architecture=architecture, )
def main(): log = logging.getLogger("zulip-provisioner") if platform.architecture()[0] == '64bit': arch = 'amd64' phantomjs_arch = 'x86_64' elif platform.architecture()[0] == '32bit': arch = "i386" phantomjs_arch = 'i686' else: log.critical("Only x86 is supported; ping [email protected] if you want another architecture.") sys.exit(1) vendor, version, codename = platform.dist() if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor]): log.critical("Unsupported platform: {} {}".format(vendor, codename)) with sh.sudo: sh.apt_get.update(**LOUD) sh.apt_get.install(*APT_DEPENDENCIES["trusty"], assume_yes=True, **LOUD) temp_deb_path = sh.mktemp("package_XXXXXX.deb", tmpdir=True) sh.wget( "{}/{}_{}_{}.deb".format( TSEARCH_URL_BASE, TSEARCH_PACKAGE_NAME["trusty"], TSEARCH_VERSION, arch, ), output_document=temp_deb_path, **LOUD ) with sh.sudo: sh.dpkg("--install", temp_deb_path, **LOUD) with sh.sudo: PHANTOMJS_PATH = "/srv/phantomjs" PHANTOMJS_BASENAME = "phantomjs-1.9.8-linux-%s" % (phantomjs_arch,) PHANTOMJS_TARBALL_BASENAME = PHANTOMJS_BASENAME + ".tar.bz2" PHANTOMJS_TARBALL = os.path.join(PHANTOMJS_PATH, PHANTOMJS_TARBALL_BASENAME) PHANTOMJS_URL = "https://bitbucket.org/ariya/phantomjs/downloads/%s" % (PHANTOMJS_TARBALL_BASENAME,) sh.mkdir("-p", PHANTOMJS_PATH, **LOUD) if not os.path.exists(PHANTOMJS_TARBALL): sh.wget(PHANTOMJS_URL, output_document=PHANTOMJS_TARBALL, **LOUD) sh.tar("xj", directory=PHANTOMJS_PATH, file=PHANTOMJS_TARBALL, **LOUD) sh.ln("-sf", os.path.join(PHANTOMJS_PATH, PHANTOMJS_BASENAME, "bin", "phantomjs"), "/usr/local/bin/phantomjs", **LOUD) with sh.sudo: sh.rm("-rf", VENV_PATH, **LOUD) sh.mkdir("-p", VENV_PATH, **LOUD) sh.chown("{}:{}".format(os.getuid(), os.getgid()), VENV_PATH, **LOUD) sh.virtualenv(VENV_PATH, **LOUD) # Add the ./tools and ./scripts/setup directories inside the repository root to # the system path; we'll reference them later. orig_path = os.environ["PATH"] os.environ["PATH"] = os.pathsep.join(( os.path.join(ZULIP_PATH, "tools"), os.path.join(ZULIP_PATH, "scripts", "setup"), orig_path )) # Put Python virtualenv activation in our .bash_profile. with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile: bash_profile.writelines([ "source .bashrc\n", "source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),), ]) # Switch current Python context to the virtualenv. activate_this = os.path.join(VENV_PATH, "bin", "activate_this.py") execfile(activate_this, dict(__file__=activate_this)) sh.pip.install(requirement=os.path.join(ZULIP_PATH, "requirements.txt"), **LOUD) with sh.sudo: sh.cp(REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH, **LOUD) # npm install and management commands expect to be run from the root of the project. os.chdir(ZULIP_PATH) sh.npm.install(**LOUD) os.system("tools/download-zxcvbn") os.system("tools/emoji_dump/build_emoji") os.system("generate_secrets.py -d") if "--travis" in sys.argv: os.system("sudo service rabbitmq-server restart") os.system("sudo service redis-server restart") os.system("sudo service memcached restart") elif "--docker" in sys.argv: os.system("sudo service rabbitmq-server restart") os.system("sudo pg_dropcluster --stop 9.3 main") os.system("sudo pg_createcluster -e utf8 --start 9.3 main") os.system("sudo service redis-server restart") os.system("sudo service memcached restart") sh.configure_rabbitmq(**LOUD) sh.postgres_init_dev_db(**LOUD) sh.do_destroy_rebuild_database(**LOUD) sh.postgres_init_test_db(**LOUD) sh.do_destroy_rebuild_test_database(**LOUD) return 0
def main(): log = logging.getLogger("zulip-provisioner") # TODO: support other architectures if platform.architecture()[0] == '64bit': arch = 'amd64' else: log.critical("Only amd64 is supported.") vendor, version, codename = platform.dist() if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor]): log.critical("Unsupported platform: {} {}".format(vendor, codename)) with sh.sudo: sh.apt_get.update(**LOUD) sh.apt_get.install(*APT_DEPENDENCIES["trusty"], assume_yes=True, **LOUD) temp_deb_path = sh.mktemp("package_XXXXXX.deb", tmpdir=True) sh.wget( "{}/{}_{}_{}.deb".format( TSEARCH_URL_BASE, TSEARCH_PACKAGE_NAME["trusty"], TSEARCH_VERSION, arch, ), output_document=temp_deb_path, **LOUD ) with sh.sudo: sh.dpkg("--install", temp_deb_path, **LOUD) with sh.sudo: PHANTOMJS_PATH = "/srv/phantomjs" PHANTOMJS_TARBALL = os.path.join(PHANTOMJS_PATH, "phantomjs-1.9.8-linux-x86_64.tar.bz2") sh.mkdir("-p", PHANTOMJS_PATH, **LOUD) sh.wget("https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2", output_document=PHANTOMJS_TARBALL, **LOUD) sh.tar("xj", directory=PHANTOMJS_PATH, file=PHANTOMJS_TARBALL, **LOUD) sh.ln("-sf", os.path.join(PHANTOMJS_PATH, "phantomjs-1.9.8-linux-x86_64", "bin", "phantomjs"), "/usr/local/bin/phantomjs", **LOUD) with sh.sudo: sh.rm("-rf", VENV_PATH, **LOUD) sh.mkdir("-p", VENV_PATH, **LOUD) sh.chown("{}:{}".format(os.getuid(), os.getgid()), VENV_PATH, **LOUD) sh.virtualenv(VENV_PATH, **LOUD) # Add the ./tools and ./scripts/setup directories inside the repository root to # the system path; we'll reference them later. orig_path = os.environ["PATH"] os.environ["PATH"] = os.pathsep.join(( os.path.join(ZULIP_PATH, "tools"), os.path.join(ZULIP_PATH, "scripts", "setup"), orig_path )) # Put Python virtualenv activation in our .bash_profile. with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile: bash_profile.writelines([ "source .bashrc\n", "source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),), ]) # Switch current Python context to the virtualenv. activate_this = os.path.join(VENV_PATH, "bin", "activate_this.py") execfile(activate_this, dict(__file__=activate_this)) sh.pip.install(requirement=os.path.join(ZULIP_PATH, "requirements.txt"), **LOUD) with sh.sudo: sh.cp(REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH, **LOUD) # npm install and management commands expect to be run from the root of the project. os.chdir(ZULIP_PATH) sh.npm.install(**LOUD) os.system("tools/download-zxcvbn") os.system("tools/emoji_dump/build_emoji") os.system("generate_secrets.py -d") if "--travis" in sys.argv: os.system("sudo service rabbitmq-server restart") os.system("sudo service redis-server restart") os.system("sudo service memcached restart") sh.configure_rabbitmq(**LOUD) sh.postgres_init_dev_db(**LOUD) sh.do_destroy_rebuild_database(**LOUD) sh.postgres_init_test_db(**LOUD) sh.do_destroy_rebuild_test_database(**LOUD)
from sh import uname, dpkg # Current kernel version. You could also get this information by reading from the file # /proc/sys/kernel/osrelease uname_output = uname('-r').strip() # like 4.4.0-79-generic kernel_version = uname_output.rsplit('-', 1)[0] print('kernel version:', kernel_version) packages_output = dpkg('-l', 'linux-image-[0-9]*', 'linux-headers-[0-9]*', _env={'COLUMNS': '200'}) packages_list = packages_output.strip().splitlines() installed_lines = filter(lambda line: line.startswith('ii'), packages_list) package_names = [line.split()[1] for line in installed_lines] remove_packages = filter(lambda package: kernel_version not in package, package_names) print('\n'.join(remove_packages))
try: import sh except ImportError: import pip pip.main(["install", "sh>=1.12"]) import sh GRAPHENE_DIR = os.path.abspath("./graphene/") # Copy config file sh.cp("config/config.py", "sgx/config.py") # Check if swig >= 3.0.10 is installed try: out = sh.grep(sh.dpkg("-s", "swig", "swig3.0"), '^Version:') version = out.split(" ")[-1].split("-")[0] except sh.ErrorReturnCode_1: version = None if not version: exit("Error: Couldn't find swig") if LooseVersion(version) < LooseVersion("3.0.10"): exit( "Error: swig version is lower than 3.0.10. Install swig 3.0.10 or higher." ) # Check if apport is installed (it produces error messages when executing Python with Graphene) try: sh.dpkg("-s", "python3-apport") exit("Error: python3-apport is installed. Please uninstall it.") except sh.ErrorReturnCode_1:
from sh import uname, sed, dpkg, awk, grep # Get our current kernel version # $(uname -r | sed -r 's/-[a-z]+//') kernel_version = sed(uname('-r'), '-r', 's/-[a-z]+//').strip() print('kernel version:', kernel_version) # get & filter package list packages = \ grep( awk( dpkg('-l', 'linux-image-[0-9]*', 'linux-headers-[0-9]*', _env={'COLUMNS': '200'}), '/ii/{print $2}' ), '-ve', kernel_version ) print(packages.strip())
def setup_networking(ob_num: int): """Sets up networking. Installs some debs manually that are required for configuring networking, since we don't yet have networking and can't apt install them. Then configures DNS and network interfaces, then bounces the network interfaces. """ print(" === Setting up networking === ") # Install network management packages manually via dpkg, since we can't apt # install them without networking already setup. print("Installing dependencies for setting up networking...") dpkg("-i", *glob("./*.deb")) print("Configuring resolved...") # Default to the Mikrotik router for DNS, with a fallback of Google's DNS sed("-i", "s/^#DNS=$/DNS=172.27.31.254 8.8.8.8/g", "/etc/systemd/resolved.conf") sed("-i", "s/^#FallbackDNS=$/FallbackDNS=8.8.8.8 8.8.4.4/g", "/etc/systemd/resolved.conf") systemctl("restart", "systemd-resolved") # Not sure if still used, but some things might be expecting orange boxen to # have this configuration file. with open("/etc/orange-box.conf", "w") as f: f.writelines([f"orangebox_number={ob_num}"]) # Disable the external ethernet port (en*) and use both of the internal # ones (enx*). The enx* interfaces map to vlan1 and vlan2, which in turn # get mapped to `172.27.{orange box #}.X` and `172.27.{orange box # + 2}.X`, # respectively. They are both bridged to the wireless network that the # orange box is connected to, hence not needing en* connected. print("Writing network configuration...") interfaces = list( sorted( Path(iface).name for iface in glob("/sys/class/net/*") if Path(iface).name.startswith("en"))) internal_ips = [f"172.27.{ob_num}.1", f"172.27.{ob_num + 2}.1"] gateway_ips = [f"172.27.{ob_num + 1}.254", f"172.27.{ob_num + 3}.254"] sh.ip("addr", "flush", "dev", interfaces[1]) sh.ifconfig(interfaces[1], f"{internal_ips[1]}/23") systemctl("stop", "NetworkManager") systemctl("disable", "NetworkManager") with open("/etc/network/interfaces", "w") as f: f.write( textwrap.dedent(f""" # These are generated by orange-box build scripts auto lo iface lo inet loopback auto {interfaces[0]} iface {interfaces[0]} inet manual auto {interfaces[1]} iface {interfaces[1]} inet manual auto {interfaces[2]} iface {interfaces[2]} inet manual auto br0 iface br0 inet static address {internal_ips[0]} netmask 255.255.254.0 gateway {gateway_ips[0]} dns-nameservers {internal_ips[0]} {gateway_ips[0]} bridge_ports {interfaces[1]} bridge_stp off bridge_fd 0 bridge_maxwait 0 auto br1 iface br1 inet static address {internal_ips[1]} netmask 255.255.254.0 bridge_ports {interfaces[2]} bridge_stp off bridge_fd 0 bridge_maxwait 0""")) print("Restarting network interfaces...") bridges = ["br0", "br1"] # Take down all of the interfaces for iface in interfaces + bridges: sh.ifdown("--force", iface) # Bring up all interfaces except enp* for iface in interfaces[1:] + bridges: sh.ifup("--force", iface) print("Waiting for network to come up...") for _ in range(60): try: ping("-c1", "8.8.8.8") break except sh.ErrorReturnCode_1: print(" - Still waiting for 8.8.8.8...") else: print("Waited too long for network to come up.") print("Please fix the network.") sys.exit(1) print("Waiting for DNS to come up...") for _ in range(60): try: ping("-c1", "launchpad.net") break except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2): print(" - Still waiting for launchpad.net...") else: print("Waited too long for DNS to come up.") print("Please fix the DNS.") sys.exit(1)
def main(): log = logging.getLogger("zulip-provisioner") # TODO: support other architectures if platform.architecture()[0] == '64bit': arch = 'amd64' else: log.critical("Only amd64 is supported.") vendor, version, codename = platform.dist() if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor]): log.critical("Unsupported platform: {} {}".format(vendor, codename)) with sh.sudo: sh.apt_get.update(**LOUD) sh.apt_get.install(*APT_DEPENDENCIES["trusty"], assume_yes=True, **LOUD) temp_deb_path = sh.mktemp("package_XXXXXX.deb", tmpdir=True) sh.wget( "{}/{}_{}_{}.deb".format( TSEARCH_URL_BASE, TSEARCH_PACKAGE_NAME["trusty"], TSEARCH_VERSION, arch, ), output_document=temp_deb_path, **LOUD ) with sh.sudo: sh.dpkg("--install", temp_deb_path, **LOUD) with sh.sudo: PHANTOMJS_PATH = "/srv/phantomjs" PHANTOMJS_TARBALL = os.path.join(PHANTOMJS_PATH, "phantomjs-1.9.8-linux-x86_64.tar.bz2") sh.mkdir("-p", PHANTOMJS_PATH, **LOUD) sh.wget("https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2", output_document=PHANTOMJS_TARBALL, **LOUD) sh.tar("xj", directory=PHANTOMJS_PATH, file=PHANTOMJS_TARBALL, **LOUD) sh.ln("-sf", os.path.join(PHANTOMJS_PATH, "phantomjs-1.9.8-linux-x86_64", "bin", "phantomjs"), "/usr/local/bin/phantomjs", **LOUD) with sh.sudo: sh.rm("-rf", VENV_PATH, **LOUD) sh.mkdir("-p", VENV_PATH, **LOUD) sh.chown("{}:{}".format(os.getuid(), os.getgid()), VENV_PATH, **LOUD) sh.virtualenv(VENV_PATH, **LOUD) # Add the ./tools and ./scripts/setup directories inside the repository root to # the system path; we'll reference them later. orig_path = os.environ["PATH"] os.environ["PATH"] = os.pathsep.join(( os.path.join(ZULIP_PATH, "tools"), os.path.join(ZULIP_PATH, "scripts", "setup"), orig_path )) # Put Python virtualenv activation in our .bash_profile. with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile: bash_profile.writelines([ "source .bashrc\n", "source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),), ]) # Switch current Python context to the virtualenv. activate_this = os.path.join(VENV_PATH, "bin", "activate_this.py") execfile(activate_this, dict(__file__=activate_this)) sh.pip.install(requirement=os.path.join(ZULIP_PATH, "requirements.txt"), **LOUD) with sh.sudo: sh.cp(REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH, **LOUD) # Add additional node packages for test-js-with-node. with sh.sudo: sh.npm.install(*NPM_DEPENDENCIES["trusty"], g=True, prefix="/usr", **LOUD) # Management commands expect to be run from the root of the project. os.chdir(ZULIP_PATH) os.system("tools/download-zxcvbn") os.system("tools/emoji_dump/build_emoji") os.system("generate_secrets.py -d") sh.configure_rabbitmq(**LOUD) sh.postgres_init_db(**LOUD) sh.do_destroy_rebuild_database(**LOUD) sh.postgres_init_test_db(**LOUD) sh.do_destroy_rebuild_test_database(**LOUD)