def GetPackages(self, project):
"""Retrieves a list of packages of a specific project.
Args:
project (str): project name.
Returns:
dict[str, str]: package names and versions as values or None if
the packages cannot be determined.
"""
# TODO: do not use builds information, it is incomplete
# instead use https://copr-be.cloud.fedoraproject.org/results/%40gift/
# testing/fedora-26-x86_64/repodata/repomd.xml
# to find primary.xml.gz or primary.sqlite.bz2
kwargs = {
'name': self._name,
'project': project}
copr_repo_url = self._COPR_REPO_URL.format(**kwargs)
download_url = '/'.join([copr_repo_url, 'repodata', 'repomd.xml'])
page_content = self._download_helper.DownloadPageContent(download_url)
if not page_content:
logging.error('Unable to retrieve repomd.xml.')
return
repomd_xml = ElementTree.fromstring(page_content)
xml_elements = repomd_xml.findall(self._PRIMARY_XML_XPATH)
if not xml_elements or not xml_elements[0].items():
logging.error('Primary data type missing from repomd.xml.')
return
href_value_tuple = xml_elements[0].items()[0]
if not href_value_tuple[1]:
logging.error('Primary data type missing from repomd.xml.')
return
download_url = '/'.join([copr_repo_url, href_value_tuple[1]])
page_content = self._download_helper.DownloadPageContent(
download_url, encoding=None)
if not page_content:
_, _, download_url = download_url.rpartition('/')
logging.error('Unable to retrieve primary.xml.gz.')
return
with gzip.GzipFile(fileobj=io.BytesIO(page_content)) as file_object:
page_content = file_object.read()
primary_xml = ElementTree.fromstring(page_content)
# Note explicitly checking xml.Element against None because of deprecation
# warning.
if primary_xml is None:
logging.error('Packages missing from primary.xml.')
return
packages = {}
for project_xml in primary_xml:
arch_xml = project_xml.find('{http://linux.duke.edu/metadata/common}arch')
if arch_xml is None or arch_xml.text != 'src':
continue
package_name_xml = project_xml.find(
'{http://linux.duke.edu/metadata/common}name')
package_version_xml = project_xml.find(
'{http://linux.duke.edu/metadata/common}version')
if package_name_xml is None or package_version_xml is None:
continue
package_name = package_name_xml.text
package_version = package_version_xml.attrib['ver']
if not package_name or not package_version:
continue
if package_name in packages:
package_version_tuple = package_version.split('.')
version_tuple = packages[package_name].split('.')
compare_result = versions.CompareVersions(
package_version_tuple, version_tuple)
if compare_result < 0:
continue
packages[package_name] = package_version
return packages
def _UninstallPackagesWindows(self, package_versions):
"""Uninstalls packages on Windows.
Args:
package_versions (dict[str, str]): versions per package.
Returns:
bool: True if the uninstall was successful.
"""
# Tuple of package name suffix, machine type, Python version.
package_info = (
('.win32.msi', 'x86', None),
('.win32-py2.7.msi', 'x86', 2),
('.win32-py3.7.msi', 'x86', 3),
('.win32-py3.8.msi', 'x86', 3),
('.win-amd64.msi', 'amd64', None),
('.win-amd64-py2.7.msi', 'amd64', 2),
('.win-amd64-py3.7.msi', 'amd64', 3),
('.win-amd64-py3.8.msi', 'amd64', 3))
connection = wmi.WMI()
query = 'SELECT PackageName FROM Win32_Product'
for product in connection.query(query):
name = getattr(product, 'PackageName', '')
name = name.lower()
has_known_suffix = False
machine_type = None
python_version = None
for name_suffix, machine_type, python_version in package_info:
has_known_suffix = name.endswith(name_suffix)
if has_known_suffix:
name = name[:-len(name_suffix)]
break
if not has_known_suffix:
continue
if (self._preferred_machine_type and
self._preferred_machine_type != machine_type):
continue
if python_version and python_version not in (2, 3):
continue
name, _, version = name.rpartition('-')
found_package = name in package_versions
version_tuple = version.split('.')
if self._force_install:
compare_result = -1
elif not found_package:
compare_result = 1
elif not package_versions[name]:
# No version was specified hence we want the package removed.
compare_result = -1
else:
compare_result = versions.CompareVersions(
version_tuple, package_versions[name])
if compare_result >= 0:
# The latest or newer version is already installed.
del package_versions[name]
if found_package and compare_result < 0:
logging.info('Removing: {0:s} {1:s}'.format(name, version))
product.Uninstall()
return True
def _GetAvailablePackages(self):
"""Determines the packages available for download.
Returns:
list[PackageDownload]: packages available for download.
"""
python_version_indicator = '-py{0:d}.{1:d}'.format(
sys.version_info[0], sys.version_info[1])
# The API is rate limited, so we scrape the web page instead.
package_urls = self._download_helper.GetPackageDownloadURLs(
preferred_machine_type=self._preferred_machine_type,
preferred_operating_system=self.operating_system)
if not package_urls:
logging.error('Unable to determine package download URLs.')
return []
# Use a dictionary so we can more efficiently set a newer version of
# a package that was set previously.
available_packages = {}
package_versions = {}
for package_url in package_urls:
_, _, package_filename = package_url.rpartition('/')
package_filename = package_filename.lower()
if not package_filename.endswith('.msi'):
# Ignore all other file extensions.
continue
# Strip off the trailing part starting with '.win'.
package_name, _, package_version = package_filename.partition('.win')
if ('-py' in package_version and
python_version_indicator not in package_version):
# Ignore packages that are for different versions of Python.
continue
if package_name.startswith('pefile-1.'):
# We need to use the most left '-' character as the separator of the
# name and the version, since version can contain the '-' character.
name, _, version = package_name.partition('-')
else:
# We need to use the most right '-' character as the separator of the
# name and the version, since name can contain the '-' character.
name, _, version = package_name.rpartition('-')
version = version.split('.')
if package_name.startswith('pefile-1.'):
last_part = version.pop()
version.extend(last_part.split('-'))
if name not in package_versions:
compare_result = 1
else:
compare_result = versions.CompareVersions(
version, package_versions[name])
if compare_result > 0:
package_versions[name] = version
package_download = PackageDownload(
name, version, package_filename, package_url)
available_packages[name] = package_download
return available_packages.values()
def _UninstallPackagesWindows(self, package_versions):
"""Uninstalls packages on Windows.
Args:
package_versions (dict[str, str]): versions per package.
Returns:
bool: True if the uninstall was successful.
"""
# Tuple of packge name suffix, machine type, Python version
package_info = (('.win32.msi', 'x86',
None), ('.win32-py2.7.msi', 'x86',
2), ('.win32-py3.6.msi', 'x86',
3), ('.win-amd64.msi', 'amd64', None),
('.win-amd64-py2.7.msi', 'amd64',
2), ('.win-amd64-py3.6.msi', 'amd64', 3))
connection = wmi.WMI()
query = 'SELECT PackageName FROM Win32_Product'
for product in connection.query(query):
name = getattr(product, 'PackageName', '')
has_known_suffix = False
machine_type = None
python_version = None
for name_suffix, machine_type, python_version in package_info:
has_known_suffix = name.endswith(name_suffix)
if has_known_suffix:
name = name[:-len(name_suffix)]
break
if not has_known_suffix:
continue
if (self._preferred_machine_type
and self._preferred_machine_type != machine_type):
continue
# TODO: improve this check to support Python 3.
if python_version and python_version != 2:
continue
name, _, version = name.rpartition('-')
found_package = name in package_versions
version_tuple = version.split('.')
if self._force_install:
compare_result = -1
elif not found_package:
compare_result = 1
else:
compare_result = versions.CompareVersions(
version_tuple, package_versions[name])
if compare_result >= 0:
# The latest or newer version is already installed.
del package_versions[name]
if not found_package and name.startswith('py'):
# Remove libyal Python packages using the old naming convention.
new_name = 'lib{0:s}-python'.format(name[2:])
found_package = new_name in package_versions
if found_package:
compare_result = -1
if found_package and compare_result < 0:
logging.info('Removing: {0:s} {1:s}'.format(name, version))
product.Uninstall()
return True
def _UninstallPackagesMacOSX(self, package_versions):
"""Uninstalls packages on Mac OS X.
Args:
package_versions (dict[str, str]): versions per package.
Returns:
bool: True if the uninstall was successful.
"""
command = '/usr/sbin/pkgutil --packages'
logging.info('Running: "{0:s}"'.format(command))
process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
if process.returncode is None:
packages, _ = process.communicate()
else:
packages = ''
if process.returncode != 0:
logging.error('Running: "{0:s}" failed.'.format(command))
return False
result = True
for package_name in packages.split('\n'):
if not package_name:
continue
matching_prefix = None
for prefix in self._PKG_NAME_PREFIXES:
if package_name.startswith(prefix):
matching_prefix = prefix
if matching_prefix:
name = package_name[len(matching_prefix):]
# Detect the PackageMaker naming convention.
if name.endswith('.pkg'):
_, _, sub_name = name[:-4].rpartition('.')
is_package_maker_pkg = True
else:
is_package_maker_pkg = False
name, _, _ = name.partition('.')
if name in package_versions:
# Determine the package version.
command = '/usr/sbin/pkgutil --pkg-info {0:s}'.format(
package_name)
logging.info('Running: "{0:s}"'.format(command))
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
shell=True)
if process.returncode is None:
package_info, _ = process.communicate()
else:
package_info = ''
if process.returncode != 0:
logging.error(
'Running: "{0:s}" failed.'.format(command))
result = False
continue
location = None
version = None
volume = None
for attribute in package_info.split('\n'):
if attribute.startswith('location: '):
_, _, location = attribute.rpartition('location: ')
elif attribute.startswith('version: '):
_, _, version = attribute.rpartition('version: ')
elif attribute.startswith('volume: '):
_, _, volume = attribute.rpartition('volume: ')
if self._force_install:
compare_result = -1
elif name not in package_versions:
compare_result = 1
elif name in ('pytsk', 'pytsk3'):
# We cannot really tell by the version number that pytsk3 needs to
# be updated, so just uninstall and update it any way.
compare_result = -1
else:
version_tuple = version.split('.')
compare_result = versions.CompareVersions(
version_tuple, package_versions[name])
if compare_result >= 0:
# The latest or newer version is already installed.
del package_versions[name]
if compare_result < 0:
# Determine the files in the package.
command = '/usr/sbin/pkgutil --files {0:s}'.format(
package_name)
logging.info('Running: "{0:s}"'.format(command))
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
shell=True)
if process.returncode is None:
package_files, _ = process.communicate()
else:
package_files = ''
if process.returncode != 0:
logging.error(
'Running: "{0:s}" failed.'.format(command))
result = False
continue
directories = []
files = []
for filename in package_files.split('\n'):
if is_package_maker_pkg:
filename = '{0:s}{1:s}/{2:s}/{3:s}'.format(
volume, location, sub_name, filename)
else:
filename = '{0:s}{1:s}'.format(
location, filename)
if os.path.isdir(filename):
directories.append(filename)
else:
files.append(filename)
logging.info('Removing: {0:s} {1:s}'.format(
name, version))
for filename in files:
if os.path.exists(filename):
os.remove(filename)
for filename in directories:
if os.path.exists(filename):
try:
os.rmdir(filename)
except OSError:
# Ignore directories that are not empty.
pass
command = '/usr/sbin/pkgutil --forget {0:s}'.format(
package_name)
exit_code = subprocess.call(command, shell=True)
if exit_code != 0:
logging.error(
'Running: "{0:s}" failed.'.format(command))
result = False
return result
def _GetPackageFilenamesAndVersions(self, package_names):
"""Determines the package filenames and versions.
Args:
package_names (list[str]): package names that should be updated
if an update is available. An empty list represents all available
packages.
Args:
tuple: contains:
dict[str, str]: filenames per package or None if no filenames could
be determined.
dict[str, str]: versions per package or None if no version could
be determined.
"""
python_version_indicator = '-py{0:d}.{1:d}'.format(
sys.version_info[0], sys.version_info[1])
# The API is rate limited, so we scrape the web page instead.
package_urls = self._download_helper.GetPackageDownloadURLs(
preferred_machine_type=self._preferred_machine_type,
preferred_operating_system=self.operating_system)
if not package_urls:
logging.error('Unable to determine package download URLs.')
return None, None
if not os.path.exists(self._download_directory):
os.mkdir(self._download_directory)
os.chdir(self._download_directory)
package_filenames = {}
package_versions = {}
for package_url in package_urls:
_, _, package_filename = package_url.rpartition('/')
if package_filename.endswith('.dmg'):
# Strip off the trailing part starting with '.dmg'.
package_name, _, _ = package_filename.partition('.dmg')
package_suffix = '.dmg'
elif package_filename.endswith('.msi'):
# Strip off the trailing part starting with '.win'.
package_name, _, package_version = package_filename.partition(
'.win')
package_suffix = '.msi'
if ('-py' in package_version
and python_version_indicator not in package_version):
# Ignore packages that are for different versions of Python.
continue
else:
# Ignore all other file exensions.
continue
if package_name.startswith('pefile-1.'):
# We need to use the most left '-' character as the separator of the
# name and the version, since version can contain the '-' character.
name, _, version = package_name.partition('-')
else:
# We need to use the most right '-' character as the separator of the
# name and the version, since name can contain the '-' character.
name, _, version = package_name.rpartition('-')
package_prefix = name
version = version.split('.')
if package_name.startswith('pefile-1.'):
last_part = version.pop()
version.extend(last_part.split('-'))
# Ignore package names if defined.
if package_names and (
(not self._exclude_packages and name not in package_names) or
(self._exclude_packages and name in package_names)):
logging.info(
'Skipping: {0:s} because it was excluded'.format(name))
continue
if name not in package_versions:
compare_result = 1
else:
compare_result = versions.CompareVersions(
version, package_versions[name])
if compare_result > 0:
package_filenames[name] = package_filename
package_versions[name] = version
if not os.path.exists(package_filename):
filenames = glob.glob('{0:s}*{1:s}'.format(
package_prefix, package_suffix))
for filename in filenames:
if os.path.isdir(filename):
continue
logging.info('Removing: {0:s}'.format(filename))
os.remove(filename)
logging.info('Downloading: {0:s}'.format(package_filename))
self._download_helper.DownloadFile(package_url)
os.chdir('..')
return package_filenames, package_versions