def GetPackages(self, project): """Retrieves a list of packages of a specific project. Args: project (str): project name. Returns: dict[str, str]: package names and versions as values or None if the packages cannot be determined. """ # TODO: do not use builds information, it is incomplete # instead use https://copr-be.cloud.fedoraproject.org/results/%40gift/ # testing/fedora-26-x86_64/repodata/repomd.xml # to find primary.xml.gz or primary.sqlite.bz2 kwargs = { 'name': self._name, 'project': project} copr_repo_url = self._COPR_REPO_URL.format(**kwargs) download_url = '/'.join([copr_repo_url, 'repodata', 'repomd.xml']) page_content = self._download_helper.DownloadPageContent(download_url) if not page_content: logging.error('Unable to retrieve repomd.xml.') return repomd_xml = ElementTree.fromstring(page_content) xml_elements = repomd_xml.findall(self._PRIMARY_XML_XPATH) if not xml_elements or not xml_elements[0].items(): logging.error('Primary data type missing from repomd.xml.') return href_value_tuple = xml_elements[0].items()[0] if not href_value_tuple[1]: logging.error('Primary data type missing from repomd.xml.') return download_url = '/'.join([copr_repo_url, href_value_tuple[1]]) page_content = self._download_helper.DownloadPageContent( download_url, encoding=None) if not page_content: _, _, download_url = download_url.rpartition('/') logging.error('Unable to retrieve primary.xml.gz.') return with gzip.GzipFile(fileobj=io.BytesIO(page_content)) as file_object: page_content = file_object.read() primary_xml = ElementTree.fromstring(page_content) # Note explicitly checking xml.Element against None because of deprecation # warning. if primary_xml is None: logging.error('Packages missing from primary.xml.') return packages = {} for project_xml in primary_xml: arch_xml = project_xml.find('{http://linux.duke.edu/metadata/common}arch') if arch_xml is None or arch_xml.text != 'src': continue package_name_xml = project_xml.find( '{http://linux.duke.edu/metadata/common}name') package_version_xml = project_xml.find( '{http://linux.duke.edu/metadata/common}version') if package_name_xml is None or package_version_xml is None: continue package_name = package_name_xml.text package_version = package_version_xml.attrib['ver'] if not package_name or not package_version: continue if package_name in packages: package_version_tuple = package_version.split('.') version_tuple = packages[package_name].split('.') compare_result = versions.CompareVersions( package_version_tuple, version_tuple) if compare_result < 0: continue packages[package_name] = package_version return packages
def _UninstallPackagesWindows(self, package_versions): """Uninstalls packages on Windows. Args: package_versions (dict[str, str]): versions per package. Returns: bool: True if the uninstall was successful. """ # Tuple of package name suffix, machine type, Python version. package_info = ( ('.win32.msi', 'x86', None), ('.win32-py2.7.msi', 'x86', 2), ('.win32-py3.7.msi', 'x86', 3), ('.win32-py3.8.msi', 'x86', 3), ('.win-amd64.msi', 'amd64', None), ('.win-amd64-py2.7.msi', 'amd64', 2), ('.win-amd64-py3.7.msi', 'amd64', 3), ('.win-amd64-py3.8.msi', 'amd64', 3)) connection = wmi.WMI() query = 'SELECT PackageName FROM Win32_Product' for product in connection.query(query): name = getattr(product, 'PackageName', '') name = name.lower() has_known_suffix = False machine_type = None python_version = None for name_suffix, machine_type, python_version in package_info: has_known_suffix = name.endswith(name_suffix) if has_known_suffix: name = name[:-len(name_suffix)] break if not has_known_suffix: continue if (self._preferred_machine_type and self._preferred_machine_type != machine_type): continue if python_version and python_version not in (2, 3): continue name, _, version = name.rpartition('-') found_package = name in package_versions version_tuple = version.split('.') if self._force_install: compare_result = -1 elif not found_package: compare_result = 1 elif not package_versions[name]: # No version was specified hence we want the package removed. compare_result = -1 else: compare_result = versions.CompareVersions( version_tuple, package_versions[name]) if compare_result >= 0: # The latest or newer version is already installed. del package_versions[name] if found_package and compare_result < 0: logging.info('Removing: {0:s} {1:s}'.format(name, version)) product.Uninstall() return True
def _GetAvailablePackages(self): """Determines the packages available for download. Returns: list[PackageDownload]: packages available for download. """ python_version_indicator = '-py{0:d}.{1:d}'.format( sys.version_info[0], sys.version_info[1]) # The API is rate limited, so we scrape the web page instead. package_urls = self._download_helper.GetPackageDownloadURLs( preferred_machine_type=self._preferred_machine_type, preferred_operating_system=self.operating_system) if not package_urls: logging.error('Unable to determine package download URLs.') return [] # Use a dictionary so we can more efficiently set a newer version of # a package that was set previously. available_packages = {} package_versions = {} for package_url in package_urls: _, _, package_filename = package_url.rpartition('/') package_filename = package_filename.lower() if not package_filename.endswith('.msi'): # Ignore all other file extensions. continue # Strip off the trailing part starting with '.win'. package_name, _, package_version = package_filename.partition('.win') if ('-py' in package_version and python_version_indicator not in package_version): # Ignore packages that are for different versions of Python. continue if package_name.startswith('pefile-1.'): # We need to use the most left '-' character as the separator of the # name and the version, since version can contain the '-' character. name, _, version = package_name.partition('-') else: # We need to use the most right '-' character as the separator of the # name and the version, since name can contain the '-' character. name, _, version = package_name.rpartition('-') version = version.split('.') if package_name.startswith('pefile-1.'): last_part = version.pop() version.extend(last_part.split('-')) if name not in package_versions: compare_result = 1 else: compare_result = versions.CompareVersions( version, package_versions[name]) if compare_result > 0: package_versions[name] = version package_download = PackageDownload( name, version, package_filename, package_url) available_packages[name] = package_download return available_packages.values()
def _UninstallPackagesWindows(self, package_versions): """Uninstalls packages on Windows. Args: package_versions (dict[str, str]): versions per package. Returns: bool: True if the uninstall was successful. """ # Tuple of packge name suffix, machine type, Python version package_info = (('.win32.msi', 'x86', None), ('.win32-py2.7.msi', 'x86', 2), ('.win32-py3.6.msi', 'x86', 3), ('.win-amd64.msi', 'amd64', None), ('.win-amd64-py2.7.msi', 'amd64', 2), ('.win-amd64-py3.6.msi', 'amd64', 3)) connection = wmi.WMI() query = 'SELECT PackageName FROM Win32_Product' for product in connection.query(query): name = getattr(product, 'PackageName', '') has_known_suffix = False machine_type = None python_version = None for name_suffix, machine_type, python_version in package_info: has_known_suffix = name.endswith(name_suffix) if has_known_suffix: name = name[:-len(name_suffix)] break if not has_known_suffix: continue if (self._preferred_machine_type and self._preferred_machine_type != machine_type): continue # TODO: improve this check to support Python 3. if python_version and python_version != 2: continue name, _, version = name.rpartition('-') found_package = name in package_versions version_tuple = version.split('.') if self._force_install: compare_result = -1 elif not found_package: compare_result = 1 else: compare_result = versions.CompareVersions( version_tuple, package_versions[name]) if compare_result >= 0: # The latest or newer version is already installed. del package_versions[name] if not found_package and name.startswith('py'): # Remove libyal Python packages using the old naming convention. new_name = 'lib{0:s}-python'.format(name[2:]) found_package = new_name in package_versions if found_package: compare_result = -1 if found_package and compare_result < 0: logging.info('Removing: {0:s} {1:s}'.format(name, version)) product.Uninstall() return True
def _UninstallPackagesMacOSX(self, package_versions): """Uninstalls packages on Mac OS X. Args: package_versions (dict[str, str]): versions per package. Returns: bool: True if the uninstall was successful. """ command = '/usr/sbin/pkgutil --packages' logging.info('Running: "{0:s}"'.format(command)) process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) if process.returncode is None: packages, _ = process.communicate() else: packages = '' if process.returncode != 0: logging.error('Running: "{0:s}" failed.'.format(command)) return False result = True for package_name in packages.split('\n'): if not package_name: continue matching_prefix = None for prefix in self._PKG_NAME_PREFIXES: if package_name.startswith(prefix): matching_prefix = prefix if matching_prefix: name = package_name[len(matching_prefix):] # Detect the PackageMaker naming convention. if name.endswith('.pkg'): _, _, sub_name = name[:-4].rpartition('.') is_package_maker_pkg = True else: is_package_maker_pkg = False name, _, _ = name.partition('.') if name in package_versions: # Determine the package version. command = '/usr/sbin/pkgutil --pkg-info {0:s}'.format( package_name) logging.info('Running: "{0:s}"'.format(command)) process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) if process.returncode is None: package_info, _ = process.communicate() else: package_info = '' if process.returncode != 0: logging.error( 'Running: "{0:s}" failed.'.format(command)) result = False continue location = None version = None volume = None for attribute in package_info.split('\n'): if attribute.startswith('location: '): _, _, location = attribute.rpartition('location: ') elif attribute.startswith('version: '): _, _, version = attribute.rpartition('version: ') elif attribute.startswith('volume: '): _, _, volume = attribute.rpartition('volume: ') if self._force_install: compare_result = -1 elif name not in package_versions: compare_result = 1 elif name in ('pytsk', 'pytsk3'): # We cannot really tell by the version number that pytsk3 needs to # be updated, so just uninstall and update it any way. compare_result = -1 else: version_tuple = version.split('.') compare_result = versions.CompareVersions( version_tuple, package_versions[name]) if compare_result >= 0: # The latest or newer version is already installed. del package_versions[name] if compare_result < 0: # Determine the files in the package. command = '/usr/sbin/pkgutil --files {0:s}'.format( package_name) logging.info('Running: "{0:s}"'.format(command)) process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) if process.returncode is None: package_files, _ = process.communicate() else: package_files = '' if process.returncode != 0: logging.error( 'Running: "{0:s}" failed.'.format(command)) result = False continue directories = [] files = [] for filename in package_files.split('\n'): if is_package_maker_pkg: filename = '{0:s}{1:s}/{2:s}/{3:s}'.format( volume, location, sub_name, filename) else: filename = '{0:s}{1:s}'.format( location, filename) if os.path.isdir(filename): directories.append(filename) else: files.append(filename) logging.info('Removing: {0:s} {1:s}'.format( name, version)) for filename in files: if os.path.exists(filename): os.remove(filename) for filename in directories: if os.path.exists(filename): try: os.rmdir(filename) except OSError: # Ignore directories that are not empty. pass command = '/usr/sbin/pkgutil --forget {0:s}'.format( package_name) exit_code = subprocess.call(command, shell=True) if exit_code != 0: logging.error( 'Running: "{0:s}" failed.'.format(command)) result = False return result
def _GetPackageFilenamesAndVersions(self, package_names): """Determines the package filenames and versions. Args: package_names (list[str]): package names that should be updated if an update is available. An empty list represents all available packages. Args: tuple: contains: dict[str, str]: filenames per package or None if no filenames could be determined. dict[str, str]: versions per package or None if no version could be determined. """ python_version_indicator = '-py{0:d}.{1:d}'.format( sys.version_info[0], sys.version_info[1]) # The API is rate limited, so we scrape the web page instead. package_urls = self._download_helper.GetPackageDownloadURLs( preferred_machine_type=self._preferred_machine_type, preferred_operating_system=self.operating_system) if not package_urls: logging.error('Unable to determine package download URLs.') return None, None if not os.path.exists(self._download_directory): os.mkdir(self._download_directory) os.chdir(self._download_directory) package_filenames = {} package_versions = {} for package_url in package_urls: _, _, package_filename = package_url.rpartition('/') if package_filename.endswith('.dmg'): # Strip off the trailing part starting with '.dmg'. package_name, _, _ = package_filename.partition('.dmg') package_suffix = '.dmg' elif package_filename.endswith('.msi'): # Strip off the trailing part starting with '.win'. package_name, _, package_version = package_filename.partition( '.win') package_suffix = '.msi' if ('-py' in package_version and python_version_indicator not in package_version): # Ignore packages that are for different versions of Python. continue else: # Ignore all other file exensions. continue if package_name.startswith('pefile-1.'): # We need to use the most left '-' character as the separator of the # name and the version, since version can contain the '-' character. name, _, version = package_name.partition('-') else: # We need to use the most right '-' character as the separator of the # name and the version, since name can contain the '-' character. name, _, version = package_name.rpartition('-') package_prefix = name version = version.split('.') if package_name.startswith('pefile-1.'): last_part = version.pop() version.extend(last_part.split('-')) # Ignore package names if defined. if package_names and ( (not self._exclude_packages and name not in package_names) or (self._exclude_packages and name in package_names)): logging.info( 'Skipping: {0:s} because it was excluded'.format(name)) continue if name not in package_versions: compare_result = 1 else: compare_result = versions.CompareVersions( version, package_versions[name]) if compare_result > 0: package_filenames[name] = package_filename package_versions[name] = version if not os.path.exists(package_filename): filenames = glob.glob('{0:s}*{1:s}'.format( package_prefix, package_suffix)) for filename in filenames: if os.path.isdir(filename): continue logging.info('Removing: {0:s}'.format(filename)) os.remove(filename) logging.info('Downloading: {0:s}'.format(package_filename)) self._download_helper.DownloadFile(package_url) os.chdir('..') return package_filenames, package_versions