def compatible_test(config, test):
# Check the test case is compatible with this platform.
if "platform" in config:
if platform.system() not in config["platform"]:
return False
# Check the test case is compatible with this distro.
if "distro" in config:
if distro.name() not in config["distro"]["name"]:
return False
if distro.version() not in config["distro"]["version"]:
return False
# Individual test can also be deactivated for a distro
# Check the test case is compatible with this platform.
if "platform" in test:
if platform.system() not in test["platform"]:
return False
# Check the test case is compatible with this distro.
if "distro" in test:
if distro.name() not in test["distro"]["name"]:
return False
if distro.version() not in test["distro"]["version"]:
return False
return True
def _collect_host_info():
host_info = {}
# collect basic host and its system information
s = platform.system().lower()
host_info['system_type'] = s
if s == 'linux':
host_info['system'] = '%s-%s' % (distro.id(), distro.version())
host_info['distro_name'] = distro.id()
host_info['distro_version'] = distro.version()
# detect isolation
host_info['isolation_type'] = 'bare-metal'
host_info['isolation'] = 'bare-metal'
if utils.is_in_docker():
host_info['isolation_type'] = 'container'
host_info['isolation'] = 'docker'
elif utils.is_in_lxc():
host_info['isolation_type'] = 'container'
host_info['isolation'] = 'lxc'
# check executors capabilities
for mod in [local_run, docker_run, lxd_run]:
caps = mod.detect_capabilities()
host_info.update(caps)
return host_info
def compatible_test(config, test):
# Check the test case is compatible with this platform.
if "platform" in config:
if platform.system() not in config["platform"]:
return False
# Check the test case is compatible with this distro.
if "distro" in config:
if distro.name() not in config["distro"]["name"]:
return False
if distro.version() not in config["distro"]["version"]:
return False
# Individual test can also be deactivated for a distro
# Check the test case is compatible with this platform.
if "platform" in test:
if platform.system() not in test["platform"]:
return False
# Check the test case is compatible with this distro.
if "distro" in test:
if distro.name() not in test["distro"]["name"]:
return False
if distro.version() not in test["distro"]["version"]:
return False
# TODO: Can we have a hybrid x86_32 and x86_64 vcvar environment?
if "platform" in config and "arch" in test:
if "Windows" in config["platform"]:
if os.environ["VSCMD_ARG_TGT_ARCH"] != test["arch"]:
return False
return True
def get_release_to_test():
try:
import distro
if distro.id() == 'ubuntu-core':
return '{}.04'.format(distro.version())
return distro.version()
except (ImportError, CalledProcessError):
import lsb_release
return lsb_release.get_distro_information()["RELEASE"]
def install_lutris(self):
self.add_apt_key(
'http://download.opensuse.org/repositories/home:/strycore/xUbuntu_%s/Release.key'
% distro.version())
self.add_apt_repo('lutris', [
'deb http://download.opensuse.org/repositories/home:/strycore/xUbuntu_%s/ ./'
% distro.version()
])
self.update_os_repo()
self.install_application('lutris')
class TestLocalProject(PackitTest):
"""Test LocalProject and save interaction with GitHub"""
def cassette_setup(self, cassette):
"""requre requires this method to be present"""
cassette.data_miner.data_type = DataTypes.Dict
@staticmethod
def commit_title(lp: LocalProject):
commit_msg = lp.git_repo.head.commit.message
return commit_msg.split("\n", 1)[0]
@pytest.mark.skipif(
distro.name() == "CentOS Stream" and distro.version() == "8",
reason=
"This test is know to fail on el8: https://github.com/packit/requre/issues/233",
)
def test_checkout_pr(self):
"""Test PR checkout with and without merging"""
project = LocalProject(
git_project=self.project,
pr_id=PR_ID,
git_url=self._project_url,
working_dir=self.static_tmp,
)
assert project.ref == f"pr/{PR_ID}"
# check that HEAD of the merge matches HEAD of main
main = project.git_repo.heads["main"]
# 'Merge pull request #231 from packit/pre-commit-ci-update-config
assert self.commit_title(project) == main.commit.message.split(
"\n", 1)[0]
assert "koji_build" in (project.working_dir /
".packit.yaml").read_text()
@pytest.mark.skipif(
distro.name() == "CentOS Stream" and distro.version() == "8",
reason=
"This test is know to fail on el8: https://github.com/packit/requre/issues/233",
)
def test_checkout_pr_no_merge(self):
"""Test PR checkout with and without merging"""
project = LocalProject(
git_project=self.project,
pr_id=PR_ID,
git_url=self._project_url,
working_dir=self.static_tmp,
merge_pr=False,
)
assert project.ref == f"pr/{PR_ID}"
assert (self.commit_title(project) ==
"Run Koji builds automatically for new dist-git commits")
assert "koji_build" in (project.working_dir /
".packit.yaml").read_text()
def get_codename_to_test():
try:
import distro
if distro.id() == 'ubuntu-core':
codename = 'focal'
if distro.version() == '18':
codename = 'bionic'
elif distro.version() == '16':
codename = 'xenial'
return codename
return distro.codename().split()[0].lower()
except (ImportError, CalledProcessError):
import lsb_release
lsb_release.get_distro_information()["CODENAME"]
def get_os_parameters():
# Gather meta information about the system
# Get OS family
platform_family = platform.system()
if platform_family == 'Windows':
from common.winutils import get_windows_data
win_version = get_windows_data()
return remove_non_ascii(
win_version["name"]), win_version["version"], platform_family
platform_id = distro.id()
platform_version = distro.version()
# In most cases this info will be valid, but there is some list of exclusions
# If it's a Darwin, this is probably Mac OS. We need to get visible version using 'sw_vers' command
if platform_id.lower() == 'darwin':
os_params = execute('sw_vers').splitlines()
platform_id = os_params[0].split(":")[1].strip()
platform_version = os_params[1].split(":")[1].strip()
return platform_id, platform_version
return platform_id, platform_version, platform_family
def test():
"""
Returns:
A dict of system information.
"""
os = platform.system()
if os == "Darwin":
return {"os": "mac", "mac_version": platform.mac_ver()[0]}
if os == "Windows":
release, version, csd, platform_type = platform.win32_ver()
return {
"os": "windows",
"windows_version_release": release,
"windows_version": version,
"windows_version_service_pack": csd,
"windows_version_os_type": platform_type,
}
if os == "Linux":
return {
"os": "linux",
"linux_distro": distro.id(),
"linux_distro_like": distro.like(),
"linux_distro_version": distro.version(),
}
return {"os": os}
def get_system_info() -> Dict[str, Any]:
"""Returns system info as a dict.
Returns:
A dict of system information.
"""
system = platform.system()
if system == "Windows":
release, version, csd, ptype = platform.win32_ver()
return {
"os": "windows",
"windows_version_release": release,
"windows_version": version,
"windows_version_service_pack": csd,
"windows_version_os_type": ptype,
}
if system == "Darwin":
return {"os": "mac", "mac_version": platform.mac_ver()[0]}
if system == "Linux":
return {
"os": "linux",
"linux_distro": distro.id(),
"linux_distro_like": distro.like(),
"linux_distro_version": distro.version(),
}
# We don't collect data for any other system.
return {"os": "unknown"}
def switch_repo(subscription, on=True):
repos_dir = '/etc/yum.repos.d'
yum_file = '{}/Rockstor-{}.repo'.format(repos_dir, subscription.name)
# Historically our base subscription url denotes our CentOS rpm repo.
subscription_distro_url = subscription.url
distro_id = distro.id()
if distro_id == 'opensuse-leap':
subscription_distro_url += '/leap/{}'.format(distro.version())
elif distro_id == 'opensuse-tumbleweed':
subscription_distro_url += '/tumbleweed'
# Check if dir /etc/yum.repos.d exists and if not create.
if not os.path.isdir(repos_dir):
# Can use os.makedirs(path) if intermediate levels also don't exist.
os.mkdir(repos_dir, )
if (on):
with open(yum_file, 'w') as rfo:
rfo.write('[Rockstor-%s]\n' % subscription.name)
rfo.write('name=%s\n' % subscription.description)
if (subscription.password is not None):
rfo.write('baseurl=http://%s:%s@%s\n' %
(subscription.appliance.uuid, subscription.password,
subscription_distro_url))
else:
rfo.write('baseurl=http://%s\n' % subscription_distro_url)
rfo.write('enabled=1\n')
rfo.write('gpgcheck=1\n')
rfo.write('gpgkey=file://%sconf/ROCKSTOR-GPG-KEY\n'
% settings.ROOT_DIR)
rfo.write('metadata_expire=1h\n')
# Set file to rw- --- --- (600) via stat constants.
os.chmod(yum_file, stat.S_IRUSR | stat.S_IWUSR)
else:
if (os.path.exists(yum_file)):
os.remove(yum_file)
def platform_info():
machine = platform.machine()
processor = platform.processor()
system = platform.system()
cpuinfo_f = '/proc/cpuinfo'
if (processor in {machine, 'unknown'} and os.path.exists(cpuinfo_f)):
with open(cpuinfo_f, 'rt') as f:
for line in f:
if line.startswith('model name'):
_, _, p = line.partition(':')
processor = p.strip()
break
if 'Linux' in system:
distribution = '{} {}'.format(distro.name(), distro.version()).strip()
else:
distribution = None
data = {
'cpu': processor,
'arch': machine,
'system': '{} {}'.format(system, platform.release()),
'distribution': distribution
}
return data
def from_pieces(cls, repo, version, commit, build_type=None):
obj = cls()
obj.repo = repo
obj.version = version
obj.commit = commit
obj.build_type = build_type
obj.system = platform.system().lower()
if obj.system == "linux":
# We recently moved from centos7 to almalinux8 as the build host for our universal
# x86_64 linux build. This changes the name of the release tarball we create.
# Unfortunately, we have a lot of hard coded references to the centos package names
# in our downsstream release code. So here we munge the name to 'centos' to keep things
# working while we fix downstream code.
# TODO(jharveymsith): Remove the almalinux to centos mapping once downstream is fixed.
if distro.id() == "centos" and distro.major_version() == "7" \
or distro.id() == "almalinux" and platform.machine().lower() == "x86_64":
obj.system = "centos"
elif distro.id == "ubuntu":
obj.system = distro.id() + distro.version()
else:
obj.system = distro.id() + distro.major_version()
if len(obj.system) == 0:
raise YBOpsRuntimeError("Cannot release on this system type: " +
platform.system())
obj.machine = platform.machine().lower()
obj.validate()
return obj
def __init__(self):
self.os_version = None
self.os_version_name = None
self.is_linux = platform.system() == "Linux"
self.linux_distro = None
self.is_windows = platform.system() == "Windows"
self.is_macos = platform.system() == "Darwin"
self.is_freebsd = platform.system() == "FreeBSD"
self.is_solaris = platform.system() == "SunOS"
if self.is_linux:
import distro
self.linux_distro = distro.id()
self.os_version = Version(distro.version())
version_name = distro.codename()
self.os_version_name = version_name if version_name != "n/a" else ""
if not self.os_version_name and self.linux_distro == "debian":
self.os_version_name = self.get_debian_version_name(self.os_version)
elif self.is_windows:
self.os_version = self.get_win_os_version()
self.os_version_name = self.get_win_version_name(self.os_version)
elif self.is_macos:
self.os_version = Version(platform.mac_ver()[0])
self.os_version_name = self.get_osx_version_name(self.os_version)
elif self.is_freebsd:
self.os_version = self.get_freebsd_version()
self.os_version_name = "FreeBSD %s" % self.os_version
elif self.is_solaris:
self.os_version = Version(platform.release())
self.os_version_name = self.get_solaris_version_name(self.os_version)
def __init__(self):
try:
self.name = distro.name()
self.version = distro.version()
except NameError:
self.name = platform.dist()[0]
self.version = platform.dist()[1]
def GetSystemInfo(self, request, context) -> SystemInfo:
system_info = SystemInfo()
system_info.python_version.major = sys.version_info.major
system_info.python_version.minor = sys.version_info.minor
system_info.python_version.micro = sys.version_info.micro
system_info.python_version.releaselevel = sys.version_info.releaselevel
system_info.python_version.serial = sys.version_info.serial
system_info.python_version.version_str = sys.version
system_info.system_family = os.name
system_info.system_family_name = platform.system()
system_info.platform = platform.platform()
for name, value in os.environ.items():
system_info.env[name] = value
system_info.uname_sysname = platform.system()
system_info.uname_nodename = platform.node()
system_info.uname_release = platform.release()
system_info.uname_version = platform.version()
system_info.uname_machine = platform.machine()
system_info.dist_name = distro.name()
system_info.dist_desc = distro.name(True)
system_info.dist_version = distro.version()
system_info.dist_id = distro.id()
return system_info
def switch_repo(subscription, on=True):
repos_dir = '/etc/yum.repos.d'
yum_file = '{}/Rockstor-{}.repo'.format(repos_dir, subscription.name)
# Historically our base subscription url denotes our CentOS rpm repo.
subscription_distro_url = subscription.url
distro_id = distro.id()
if distro_id == 'opensuse-leap':
subscription_distro_url += '/leap/{}'.format(distro.version())
elif distro_id == 'opensuse-tumbleweed':
subscription_distro_url += '/tumbleweed'
# Check if dir /etc/yum.repos.d exists and if not create.
if not os.path.isdir(repos_dir):
# Can use os.makedirs(path) if intermediate levels also don't exist.
os.mkdir(repos_dir, )
if (on):
with open(yum_file, 'w') as rfo:
rfo.write('[Rockstor-%s]\n' % subscription.name)
rfo.write('name=%s\n' % subscription.description)
if (subscription.password is not None):
rfo.write('baseurl=http://%s:%s@%s\n' %
(subscription.appliance.uuid, subscription.password,
subscription_distro_url))
else:
rfo.write('baseurl=http://%s\n' % subscription_distro_url)
rfo.write('enabled=1\n')
rfo.write('gpgcheck=1\n')
rfo.write('gpgkey=file://%sconf/ROCKSTOR-GPG-KEY\n' %
settings.ROOT_DIR)
rfo.write('metadata_expire=1h\n')
# Set file to rw- --- --- (600) via stat constants.
os.chmod(yum_file, stat.S_IRUSR | stat.S_IWUSR)
else:
if (os.path.exists(yum_file)):
os.remove(yum_file)
def os_release():
"""
This code detects your os with the distro module and return the name and version. If it is not detected correctly it
returns "unknown" (str) and "0" (float).
@:returns tuple (str, float)
WHERE
str is the name
int is the version number
"""
distroname = distro.id()
distrolike = distro.like()
version = distro.version()
redhat = ["centos", "fedora", "rhel"]
if distroname in redhat or distrolike in redhat:
if distroname in ["centos", "fedora"]:
return distroname, float(version)
else:
return "redhat", float(version)
if distroname is ["debian", "ubuntu"]:
return distroname, float(version)
if distrolike == "suse":
return "suse", float(version)
return "unknown", 0.0
def check_distribution():
'''
Check if the distribution is supported by the installer.
:return: The codename of the distribution
'''
bionic_code_names = ['bionic', 'tara', 'tessa', 'tina', 'disco']
debian_code_names = ['buster', 'stretch', 'kali-rolling']
focal_code_names = ['focal', 'ulyana', 'ulyssa', 'uma']
codename = distro.codename().lower()
if codename in bionic_code_names:
logging.debug('Ubuntu 18.04 detected')
return 'bionic'
if codename in focal_code_names:
logging.debug('Ubuntu 20.04 detected')
return 'focal'
if codename in debian_code_names:
logging.debug('Debian/Kali detected')
return 'debian'
if distro.id() == 'fedora':
logging.debug('Fedora detected')
return 'fedora'
logging.critical(
'Your Distribution ({} {}) is not supported. FACT Installer requires Ubuntu 18.04, 20.04 or compatible!'
.format(distro.id(), distro.version()))
sys.exit(1)
def __init__(self):
self.os_version = None
self.os_version_name = None
self.is_linux = platform.system() == "Linux"
self.linux_distro = None
self.is_windows = platform.system() == "Windows"
self.is_macos = platform.system() == "Darwin"
self.is_freebsd = platform.system() == "FreeBSD"
self.is_solaris = platform.system() == "SunOS"
if self.is_linux:
import distro
self.linux_distro = distro.id()
self.os_version = Version(distro.version())
version_name = distro.codename()
self.os_version_name = version_name if version_name != "n/a" else ""
if not self.os_version_name and self.linux_distro == "debian":
self.os_version_name = self.get_debian_version_name(
self.os_version)
elif self.is_windows:
self.os_version = self.get_win_os_version()
self.os_version_name = self.get_win_version_name(self.os_version)
elif self.is_macos:
self.os_version = Version(platform.mac_ver()[0])
self.os_version_name = self.get_osx_version_name(self.os_version)
elif self.is_freebsd:
self.os_version = self.get_freebsd_version()
self.os_version_name = "FreeBSD %s" % self.os_version
elif self.is_solaris:
self.os_version = Version(platform.release())
self.os_version_name = self.get_solaris_version_name(
self.os_version)
def __init__(self) -> None:
"""
Creates a new HostDistribution instance.
"""
try:
import distro
self._id = distro.id()
self._name = distro.name(pretty=False)
self._fullname = distro.name(pretty=True)
try:
self._major = int(distro.major_version())
except ValueError:
self._major = -1
try:
self._minor = int(distro.minor_version())
except ValueError:
self._minor = -1
try:
self._build = int(distro.build_number())
except ValueError:
self._build = -1
self._versionstring = distro.version(pretty=False)
self._codename = distro.codename()
self._like = distro.like()
except ImportError:
from suisei.murasame.exceptions import MissingRequirementError
raise MissingRequirementError(
'HostDistribution requires the distro package.',
requirement='distro')
def _get_current_release(organization, override_version):
connection = get_connection(organization)
client = connection.get_client(
'azext_devops.dev.common.client_tool.client_tool_client.ClientToolClient'
)
logger.debug("Looking up current version of ArtifactTool...")
# Distro returns empty strings on Windows currently, so don't even send
distro_name = distro.id() or None
distro_version = distro.version() or None
os_name = platform.system()
arch = platform.machine()
# For M1 macs, there is no version of artifact tool. However, the x86_64
# version can run under Rosetta, so we use that instead.
if os_name == "Darwin" and arch in ["amd64", "arm64"]:
arch = "x86_64"
# Similarly for Windows ARM64 targets there is no version of artifact tool. However, the x86_64
# version can run under emulation, so we use that instead.
if os_name == "Windows" and arch == "ARM64":
arch = "x86_64"
release = client.get_clienttool_release("ArtifactTool",
os_name=os_name,
arch=arch,
distro_name=distro_name,
distro_version=distro_version,
version=override_version)
return (release.uri, _compute_id(release)) if release is not None else None
def _system_info():
system = platform.system()
if system == "Windows":
version = sys.getwindowsversion()
return {
"os": "windows",
"windows_version_build": version.build,
"windows_version_major": version.major,
"windows_version_minor": version.minor,
"windows_version_service_pack": version.service_pack,
}
if system == "Darwin":
return {"os": "mac", "mac_version": platform.mac_ver()[0]}
if system == "Linux":
return {
"os": "linux",
"linux_distro": distro.id(),
"linux_distro_like": distro.like(),
"linux_distro_version": distro.version(),
}
# We don't collect data for any other system.
raise NotImplementedError
def _get_distro_info() -> Tuple[str, str, str, str]:
if psutil.WINDOWS:
return "windows", platform.system(), platform.release(), ""
elif psutil.OSX:
import plistlib
with open("/System/Library/CoreServices/SystemVersion.plist",
"rb") as f:
sw_vers = plistlib.load(f)
return "mac", sw_vers["ProductName"], sw_vers["ProductVersion"], ""
elif _is_android():
import subprocess
android_version = subprocess.run(
["getprop", "ro.build.version.release"],
check=True,
stdout=subprocess.PIPE,
).stdout
return "android", "Android", android_version.decode().strip(), ""
elif psutil.LINUX:
import distro
return distro.id(), distro.name(), distro.version(), distro.codename()
raise NotImplementedError("unsupported OS")
def upload(name, asm, compilers, compiler_args, strip):
conn = DB()
if conn:
cursor = conn.cursor()
# Upsert assembly contents to the `assembly' table.
with open(asm, "r") as f:
contents = f.read()
checksum = hashlib.md5(contents.encode("utf-8")).hexdigest()
cursor.execute(
"""
INSERT INTO assembly (checksum, content)
VALUES (%s, %s)
ON CONFLICT (checksum)
DO UPDATE SET updated_at = NOW()
RETURNING assembly_id
""",
(checksum, contents),
)
assembly_id = cursor.fetchone()[0]
# Insert disassembly details into the `disassembled' table.
cursor.execute(
"""
INSERT INTO disassembled (
name,
assembly_id,
compiler,
compiler_args,
platform,
distro,
ci_job_image,
ci_pipeline_id,
ci_commit_sha,
ci_commit_before_sha,
ci_commit_branch,
ci_commit_ref_slug,
strip
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""",
(
name,
assembly_id,
" ".join(compilers),
" ".join(compiler_args),
platform.system(),
" ".join([distro.name(), distro.version()]),
os.environ.get("CI_JOB_IMAGE"),
os.environ.get("CI_PIPELINE_ID"),
os.environ.get("CI_COMMIT_SHA"),
os.environ.get("CI_COMMIT_BEFORE_SHA"),
os.environ.get("CI_COMMIT_BRANCH"),
os.environ.get("CI_COMMIT_REF_SLUG"),
strip,
),
)
conn.commit()
def _get_linux_distro_info(self):
import distro
self.linux_distro = distro.id()
self.os_version = Version(distro.version())
version_name = distro.codename()
self.os_version_name = version_name if version_name != "n/a" else ""
if not self.os_version_name and self.linux_distro == "debian":
self.os_version_name = self.get_debian_version_name(self.os_version)
def _collect_linux(self):
import distro
info = {}
info[self.PARAM_OS] = 'linux'
info[self.PARAM_LINUX_DISTRO] = distro.id()
info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
return info
def send_distroinfo(self):
while self.start:
data = {"distro": self.os_distro_name, "version": distro.version()}
self.emit("distro_info", {
"key": "sysinfo:distro_info",
"data": data
})
gevent.sleep(600)
def send_distroinfo(self):
while self.start:
data = {'distro': self.os_distro_name, 'version': distro.version()}
self.emit('distro_info', {
'key': 'sysinfo:distro_info',
'data': data
})
gevent.sleep(600)
def get_values(self):
if sys.platform == "darwin":
return {"os": {"id": "macos", "version": platform.mac_ver()[0]}}
elif sys.platform == "win32":
return {"os": {"id": "windows", "version": platform.win32_ver()[1]}}
elif distro is not None:
return {"os": {"id": distro.id(), "version": distro.version(best=True)}}
return {}
def get_platforms():
"""
Returns a list of increasingly-generic identifiers for the current
system.
"""
global _platforms
if _platforms is not None:
return _platforms
# Start with the most generic - the OS
system = platform.system().lower()
# Initialize list with the system.
_platforms = [system]
# OS-specific stuff
if system == "linux":
import distro
dist_id = distro.id()
_platforms.insert(0, dist_id)
if dist_id == "ubuntu":
# Ubuntu "minor" versions are distinct, eg., Ubuntu 16.10
# is potentially quite different from Ubuntu 16.04. So we
# want to use the combined version.
dist_ver = distro.version()
else:
# Other supported distros use rolling releases, so eg.
# Centos 6.5 shouldn't differ importantly from Centos 6.9.
# Use only the major version number.
dist_ver = distro.major_version()
_platforms.insert(0, f"{dist_id}{dist_ver}")
_platforms.insert(0, f"{dist_id}-{dist_ver}")
if dist_id == "sles" or dist_id.startswith("opensuse"):
# Cbdeps 1.0, at least, refers to all SUSE as "suse", so offer
# those as platform names too
dist_id = "suse"
_platforms.insert(0, dist_id)
_platforms.insert(0, f"{dist_id}{dist_ver}")
_platforms.insert(0, f"{dist_id}-{dist_ver}")
elif system == "darwin":
_platforms.insert(0, "macosx")
_platforms.insert(0, "macos")
_platforms.insert(0, "mac")
_platforms.insert(0, "osx")
elif system == "windows":
# QQQ Somehow introspect MSVC version?
_platforms.insert(0, "windows_msvc2015")
_platforms.insert(0, "windows_msvc2017")
_platforms.insert(0, "win")
return _platforms
def get_os_version():
"""Determine system version."""
os_version = None
# platform.dist was removed in Python 3.8
if hasattr(platform, 'dist'):
os_version = platform.dist()[1]
# take into account that on some OSs, platform.dist returns an empty string as OS version,
# for example on OpenSUSE Leap 15.2
if not os_version and HAVE_DISTRO:
os_version = distro.version()
if not os_version and os.path.exists(ETC_OS_RELEASE):
os_release_txt = read_file(ETC_OS_RELEASE)
version_regex = re.compile('^VERSION="?(?P<version>[^"\n]+)"?$', re.M)
res = version_regex.search(os_release_txt)
if res:
os_version = res.group('version')
else:
# VERSION may not always be defined (for example on Gentoo),
# fall back to VERSION_ID in that case
version_regex = re.compile('^VERSION_ID="?(?P<version>[^"\n]+)"?$', re.M)
res = version_regex.search(os_release_txt)
if res:
os_version = res.group('version')
if os_version:
# older SLES subversions can only be told apart based on kernel version,
# see http://wiki.novell.com/index.php/Kernel_versions
sles_version_suffixes = {
'11': [
('2.6.27', ''),
('2.6.32', '_SP1'),
('3.0.101-63', '_SP4'),
# not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough?
('3.0.101', '_SP3'),
# SP2 kernel versions range from 3.0.13 - 3.0.101
('3.0', '_SP2'),
],
'12': [
('3.12.28', ''),
('3.12.49', '_SP1'),
],
}
if get_os_name() in ['suse', 'SLES'] and os_version in sles_version_suffixes:
# append suitable suffix to system version
kernel_version = platform.uname()[2]
for (kver, suff) in sles_version_suffixes[os_version]:
if kernel_version.startswith(kver):
os_version += suff
break
return os_version
else:
return UNKNOWN
def get_distro_project():
try:
import distro
project = '{0}-{1}'.format(distro.name(), distro.version())
except ImportError:
project = '-'.join(platform.linux_distribution()[0:2])
return slugify(project)
def update_check(subscription=None):
if (subscription is not None):
switch_repo(subscription)
pkg = 'rockstor'
version, date = rpm_build_info(pkg)
if date is None:
# None date signifies no rpm installed so list all changelog entries.
date = 'all'
log = False
available = False
new_version = None
updates = []
try:
o, e, rc = run_command([YUM, 'changelog', date, pkg])
except CommandException as e:
# Catch as yet unconfigured repos ie Leap 15.1: error log accordingly.
# Avoids breaking current version display and update channel selection.
emsg = 'Error\\: Cannot retrieve repository metadata \\(repomd.xml\\)'
if re.match(emsg, e.err[-2]) is not None:
logger.error('Rockstor repo for distro.id ({}) version ({}) may '
'not exist: pending or deprecated.\nReceived: ({}).'
.format(distro.id(), distro.version(), e.err))
new_version = version # Explicitly set (flag) for code clarity.
return version, new_version, updates
# otherwise we raise an exception as normal.
raise e
for l in o:
if (re.search('Available Packages', l) is not None):
available = True
if (not available):
continue
if (new_version is None and (re.match('rockstor-', l) is not None)):
new_version = l.split()[0].split(
'rockstor-')[1].split('.x86_64')[0]
if (log is True):
updates.append(l)
if (len(l.strip()) == 0):
log = False
if (re.match('\* ', l) is not None):
log = True
if (new_version is None):
new_version = version
# do a second check which is valid for updates without changelog
# updates. eg: same day updates, testing updates.
o, e, rc = run_command([YUM, 'update', pkg, '--assumeno'], throw=False)
if (rc == 1):
for l in o:
if (re.search('will be an update', l) is not None):
if (re.search('rockstor.x86_64', l) is not None):
new_version = l.strip().split()[3].split(':')[1]
return version, new_version, updates
def print_python():
from platform import architecture, mac_ver, uname, win32_ver
if 'intelpython' in sys.executable:
t = 'Intel-'
# elif 'PyPy ' in sys.version:
# t = 'PyPy-'
else:
t = ''
t = '%sPython: %s (%s)' % (t, sys.version.split()[0], architecture()[0])
if win32_ver()[0]:
t = t, 'Windows', win32_ver()[0]
elif mac_ver()[0]:
t = t, ('iOS' if sys.platform == 'ios' else 'macOS'), mac_ver()[0]
else:
try:
import distro # <http://GitHub.com/nir0s/distro>
t = t, bytes_to_str(distro.name()), bytes_to_str(distro.version())
except ImportError:
t = (t,) + uname()[0:3:2]
print(' '.join(t))
def _default_platform_name(distutils_util_get_platform):
"""Guess a sane default platform name.
On OS X and Windows, just uses the default platform name. On Linux, uses
information from the `platform` module to try to make something reasonable.
"""
def grab_version(string, num):
"""Grab the `num` most significant components of a version string.
>>> grab_version('12.04.1', 2)
'12.04'
>>> grab_version('8.2', 1)
'8'
"""
return '.'.join(string.split('.')[:num])
if platform.system() == 'Linux':
dist, version = distro.id(), distro.version()
dist = re.sub('linux$', '', dist.lower()).strip()
# Try to determine a good "release" name. This is highly dependent on
# distribution and what guarantees they provide between versions.
release = None
if dist in {'debian', 'rhel', 'centos', 'fedora', 'opensuse'}:
release = grab_version(version, 1) # one version component
elif dist in {'ubuntu', 'amzn'}:
release = grab_version(version, 2) # two version components
if release:
return 'linux_{dist}_{release}_{arch}'.format(
dist=_sanitize_platform(dist),
release=_sanitize_platform(release),
arch=_sanitize_platform(platform.machine()),
)
# For Windows, OS X, or Linux distributions we couldn't identify, just fall
# back to whatever pip normally uses.
return _sanitize_platform(distutils_util_get_platform())
def write_configuration(directory, sort_packages, find_inputs_outputs,
overwrite=False):
"""Writes the canonical YAML configuration file.
"""
database = directory / 'trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
# Reads info from database
files, inputs, outputs = get_files(conn)
# Identifies which file comes from which package
if sort_packages:
files, packages = identify_packages(files)
else:
packages = []
# Writes configuration file
config = directory / 'config.yml'
distribution = [distro.id(), distro.version()]
cur = conn.cursor()
if overwrite or not config.exists():
runs = []
# This gets all the top-level processes (p.parent ISNULL) and the first
# executed file for that process (sorting by ids, which are
# chronological)
executions = cur.execute(
'''
SELECT e.name, e.argv, e.envp, e.workingdir,
p.timestamp, p.exit_timestamp, p.exitcode
FROM processes p
JOIN executed_files e ON e.id=(
SELECT id FROM executed_files e2
WHERE e2.process=p.id
ORDER BY e2.id
LIMIT 1
)
WHERE p.parent ISNULL;
''')
else:
# Loads in previous config
runs, oldpkgs, oldfiles = load_config(config,
canonical=False,
File=TracedFile)
# Same query as previous block but only gets last process
executions = cur.execute(
'''
SELECT e.name, e.argv, e.envp, e.workingdir,
p.timestamp, p.exit_timestamp, p.exitcode
FROM processes p
JOIN executed_files e ON e.id=(
SELECT id FROM executed_files e2
WHERE e2.process=p.id
ORDER BY e2.id
LIMIT 1
)
WHERE p.parent ISNULL
ORDER BY p.id
LIMIT 2147483647 OFFSET ?;
''',
(len(runs),))
for (r_name, r_argv, r_envp, r_workingdir,
r_start, r_end, r_exitcode) in executions:
# Decodes command-line
argv = r_argv.split('\0')
if not argv[-1]:
argv = argv[:-1]
# Decodes environment
envp = r_envp.split('\0')
if not envp[-1]:
envp = envp[:-1]
environ = dict(v.split('=', 1) for v in envp)
run = {'id': "run%d" % len(runs),
'binary': r_name, 'argv': argv,
'workingdir': unicode_(Path(r_workingdir)),
'architecture': platform.machine().lower(),
'distribution': distribution,
'hostname': platform.node(),
'system': [platform.system(), platform.release()],
'environ': environ,
'uid': os.getuid(),
'gid': os.getgid()}
if r_exitcode & 0x0100:
run['signal'] = r_exitcode & 0xFF
else:
run['exitcode'] = r_exitcode & 0xFF
if r_end is not None:
run['walltime'] = (r_end - r_start) / 1.0E9 # ns to s
runs.append(run)
cur.close()
conn.close()
if find_inputs_outputs:
inputs_outputs = compile_inputs_outputs(runs, inputs, outputs)
else:
inputs_outputs = {}
save_config(config, runs, packages, files, reprozip_version,
inputs_outputs)
print("Configuration file written in {0!s}".format(config))
print("Edit that file then run the packer -- "
"use 'reprozip pack -h' for help")
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import distro
print 'os_release_info: {0}'.format(distro.os_release_info())
print 'lsb_release_info: {0}'.format(distro.lsb_release_info())
print 'distro_release_info: {0}'.format(distro.distro_release_info())
print 'id: {0}'.format(distro.id())
print 'name: {0}'.format(distro.name())
print 'name_pretty: {0}'.format(distro.name(True))
print 'version: {0}'.format(distro.version())
print 'version_pretty: {0}'.format(distro.version(True))
print 'like: {0}'.format(distro.like())
print 'codename: {0}'.format(distro.codename())
print 'linux_distribution_full: {0}'.format(distro.linux_distribution())
print 'linux_distribution: {0}'.format(distro.linux_distribution(False))
print 'major_version: {0}'.format(distro.major_version())
print 'minor_version: {0}'.format(distro.minor_version())
print 'build_number: {0}'.format(distro.build_number())
def send_distroinfo(self):
while self.start:
data = {'distro': self.os_distro_name, 'version': distro.version()}
self.emit('distro_info',
{'key': 'sysinfo:distro_info', 'data': data})
gevent.sleep(600)
def _print_package_info(pack, info, verbosity=1):
print("Pack file: %s" % pack)
print("\n----- Pack information -----")
print("Compressed size: %s" % hsize(pack.size()))
info_pack = info.get('pack')
if info_pack:
if 'total_size' in info_pack:
print("Unpacked size: %s" % hsize(info_pack['total_size']))
if 'total_paths' in info_pack:
print("Total packed paths: %d" % info_pack['total_paths'])
if verbosity >= 3:
print(" Files: %d" % info_pack['files'])
print(" Directories: %d" % info_pack['dirs'])
if info_pack.get('symlinks'):
print(" Symbolic links: %d" % info_pack['symlinks'])
if info_pack.get('hardlinks'):
print(" Hard links: %d" % info_pack['hardlinks'])
if info_pack.get('others'):
print(" Unknown (what!?): %d" % info_pack['others'])
print("\n----- Metadata -----")
info_meta = info['meta']
if verbosity >= 3:
print("Total paths: %d" % info_meta['total_paths'])
print("Listed packed paths: %d" % info_meta['packed_paths'])
if info_meta.get('packages'):
print("Total software packages: %d" % info_meta['packages'])
print("Packed software packages: %d" % info_meta['packed_packages'])
if verbosity >= 3:
print("Files from packed software packages: %d" %
info_meta['packed_packages_files'])
print("Files from unpacked software packages: %d" %
info_meta['unpacked_packages_files'])
if 'architecture' in info_meta:
print("Architecture: %s (current: %s)" % (info_meta['architecture'],
platform.machine().lower()))
if 'distribution' in info_meta:
distribution = ' '.join(t for t in info_meta['distribution'] if t)
current_distribution = [distro.id(), distro.version()]
current_distribution = ' '.join(t for t in current_distribution if t)
print("Distribution: %s (current: %s)" % (
distribution, current_distribution or "(not Linux)"))
if 'runs' in info:
runs = info['runs']
print("Runs (%d):" % len(runs))
for run in runs:
cmdline = ' '.join(shell_escape(a) for a in run['argv'])
if len(runs) == 1 and run['id'] == "run0":
print(" %s" % cmdline)
else:
print(" %s: %s" % (run['id'], cmdline))
if verbosity >= 2:
print(" wd: %s" % run['workingdir'])
if 'signal' in run:
print(" signal: %d" % run['signal'])
else:
print(" exitcode: %d" % run['exitcode'])
if run.get('walltime') is not None:
print(" walltime: %s" % run['walltime'])
inputs_outputs = info.get('inputs_outputs')
if inputs_outputs:
if verbosity < 2:
print("Inputs/outputs files (%d): %s" % (
len(inputs_outputs), ", ".join(sorted(inputs_outputs))))
else:
print("Inputs/outputs files (%d):" % len(inputs_outputs))
for name, f in sorted(iteritems(inputs_outputs)):
t = []
if f['read_runs']:
t.append("in")
if f['write_runs']:
t.append("out")
print(" %s (%s): %s" % (name, ' '.join(t), f['path']))
unpacker_status = info.get('unpacker_status')
if unpacker_status:
print("\n----- Unpackers -----")
for s, n in [(COMPAT_OK, "Compatible"), (COMPAT_MAYBE, "Unknown"),
(COMPAT_NO, "Incompatible")]:
if s != COMPAT_OK and verbosity < 2:
continue
if s not in unpacker_status:
continue
upks = unpacker_status[s]
print("%s (%d):" % (n, len(upks)))
for upk_name, msg in upks:
if msg is not None:
print(" %s (%s)" % (upk_name, msg))
else:
print(" %s" % upk_name)
def platform_profiles(self):
if platform.system() == 'Darwin':
atoms = set(['darwin'])
# detect available macos package managers
if os.system('which brew >/dev/null') == 0:
atoms.add('brew')
self.platform = Brew()
return ["platform:%s" % (atom,) for atom in sorted(atoms)]
distro_id = distro.id()
if not distro_id:
log = logging.getLogger(__name__)
log.error('Unable to determine distro ID. '
'Does /etc/os-release exist or '
'is lsb_release installed?')
raise Exception('Distro name not found')
# NOTE(toabctl): distro can be more than one string (i.e. "SUSE LINUX")
codename = distro.codename().lower()
release = distro.version().lower()
# NOTE(toabctl): space is a delimiter for bindep, so remove the spaces
distro_id = "".join(distro_id.split()).lower()
atoms = set([distro_id])
atoms.update(self.codenamebits(distro_id, codename))
atoms.update(self.releasebits(distro_id, release))
if distro_id in ["debian", "ubuntu"]:
atoms.add("dpkg")
self.platform = Dpkg()
# RPM distros seem to be especially complicated
elif distro_id in ["amzn", "amazonami",
"centos", "rhel",
"redhatenterpriseserver",
"redhatenterpriseworkstation",
"fedora",
"opensuseproject", "opensuse", "opensuse-leap",
"opensuse-tumbleweed", "sles", "suselinux"]:
# Distro aliases
if distro_id in ["redhatenterpriseserver",
"redhatenterpriseworkstation"]:
# just short alias
atoms.add("rhel")
atoms.update(self.codenamebits("rhel", codename))
atoms.update(self.releasebits("rhel", release))
elif distro_id == 'rhel' and 'server' in distro.name().lower():
atoms.add("redhatenterpriseserver")
atoms.update(self.codenamebits("redhatenterpriseserver",
codename))
atoms.update(self.releasebits("redhatenterpriseserver",
release))
elif (distro_id == 'rhel' and
'workstation' in distro.name().lower()):
atoms.add("redhatenterpriseworkstation")
atoms.update(self.codenamebits("redhatenterpriseworkstation",
codename))
atoms.update(self.releasebits("redhatenterpriseworkstation",
release))
elif "amzn" in distro_id:
atoms.add("amazonami")
atoms.update(self.codenamebits("amazonami", codename))
atoms.update(self.releasebits("amazonami", release))
elif "amazonami" in distro_id:
atoms.add("amzn")
atoms.update(self.codenamebits("amzn", codename))
atoms.update(self.releasebits("amzn", release))
elif "opensuse" in distro_id:
# just short alias
atoms.add("opensuse")
atoms.update(self.codenamebits("opensuse", codename))
atoms.update(self.releasebits("opensuse", release))
atoms.add("opensuseproject")
atoms.update(self.codenamebits("opensuseproject", codename))
atoms.update(self.releasebits("opensuseproject", release))
elif "sles" in distro_id:
atoms.add("suselinux")
atoms.update(self.codenamebits("suselinux", codename))
atoms.update(self.releasebits("suselinux", release))
elif "suselinux" in distro_id:
atoms.add("sles")
atoms.update(self.codenamebits("sles", codename))
atoms.update(self.releasebits("sles", release))
# Family aliases
if 'suse' in distro_id or distro_id == 'sles':
atoms.add("suse")
else:
atoms.add("redhat")
atoms.add("rpm")
self.platform = Rpm()
elif distro_id in ["gentoo"]:
atoms.add("emerge")
self.platform = Emerge()
elif distro_id in ["arch"]:
atoms.add("pacman")
self.platform = Pacman()
elif distro_id in ["alpine"]:
atoms.add("apk")
self.platform = Apk()
else:
self.platform = Unknown()
return ["platform:%s" % (atom,) for atom in sorted(atoms)]
cmd("""LC_ALL=C.UTF-8 add-apt-repository -y ppa:ondrej/apache2 &&
apt-key update &&
apt-get -y -q update &&
apt-get -y -q install apache2""")
else:
# debian 只有低版本的可以用
cmd("apt-get -y -q install apache2")
cmd("""a2enmod rewrite mime include headers filter expires deflate autoindex setenvif ssl""")
if not cmd("a2enmod http2", allow_failure=True):
warnprint("[Warning!] your server does not support http2")
sleep(0.5)
# (可选) 更新一下各种包
if not (distro.id() == 'ubuntu' and distro.version() == '14.04'): # 系统不是ubuntu 14.04
# Ubuntu 14.04 执行本命令的时候会弹一个postfix的交互, 所以不执行
cmd('apt-get -y -q upgrade', allow_failure=True)
cmd("""apt-get -y -q install libapache2-mod-wsgi-py3&& a2enmod wsgi""")
# 安装和更新必须的python包
cmd('python3 -m pip install -U flask')
cmd('python3 -m pip install requests==2.11.0')
# 安装和更新非必须, 但是有好处的python包, 允许失败
cmd('python3 -m pip install -U chardet', allow_failure=True)
cmd("python3 -m pip install -U cchardet", allow_failure=True)
cmd("python3 -m pip install -U fastcache", allow_failure=True)
cmd("python3 -m pip install -U lru-dict", allow_failure=True)
