def _download_apps(self, app_cache): filenames = ['index.json.gz'] if not ucr_is_false('appcenter/index/verify'): filenames.append('index.json.gz.gpg') self._download_files(app_cache, filenames) json_apps = self._load_index_json(app_cache) files_to_download, something_changed_remotely = self._read_index_json(app_cache.get_cache_dir(), json_apps) num_files_to_be_downloaded = len(files_to_download) self.log('%d file(s) are new' % num_files_to_be_downloaded) num_files_threshold = 5 if num_files_to_be_downloaded > num_files_threshold: files_to_download = self._download_archive(app_cache, files_to_download) threads = [] max_threads = 10 files_per_thread = max(num_files_threshold, int(ceil(float(len(files_to_download)) / max_threads))) while files_to_download: # normally, this should be only one thread as # _download_archive() is used if many files are to be downloaded # but if all.tar.gz fails, everything needs to be downloaded # don't do this at once as this opens 100 connections. files_to_download_in_thread, files_to_download = files_to_download[:files_per_thread], files_to_download[files_per_thread:] self.log('Starting to download %d file(s) directly' % len(files_to_download_in_thread)) thread = Thread(target=self._download_directly, args=(app_cache, files_to_download_in_thread,)) thread.start() threads.append(thread) time.sleep(0.1) # wait 100 milliseconds so that not all threads start at the same time for thread in threads: thread.join() return something_changed_remotely
def _download_apps(self, app_cache): filenames = ['index.json.gz'] if not ucr_is_false('appcenter/index/verify'): filenames.append('index.json.gz.gpg') filenames.append('all.tar.gpg') if self._download_files(app_cache, filenames): appcenter_host = app_cache.get_server() if appcenter_host.startswith('https'): appcenter_host = 'http://%s' % appcenter_host[8:] all_tar_file = os.path.join(app_cache.get_cache_dir(), '.all.tar') all_tar_url = '%s/meta-inf/%s/all.tar.zsync' % (appcenter_host, app_cache.get_ucs_version()) self.log('Downloading "%s"...' % all_tar_url) cwd = os.getcwd() os.chdir(os.path.dirname(all_tar_file)) try: if self._subprocess(['zsync', all_tar_url, '-q', '-o', all_tar_file]).returncode: # fallback: download all.tar.gz without zsync. some proxys have difficulties with it, including: # * Range requests are not supported # * HTTP requests are altered self.warn('Downloading the App archive via zsync failed. Falling back to download it directly.') self.warn('For better performance, try to make zsync work for "%s". The error may be caused by a proxy altering HTTP requests' % all_tar_url) self._download_files(app_cache, ['all.tar.gz']) self._uncompress_archive(app_cache, os.path.join(app_cache.get_cache_dir(), '.all.tar.gz')) finally: os.chdir(cwd) self._verify_file(all_tar_file) self._extract_archive(app_cache) return True return False
def verify(self): if ucr_is_false('appcenter/index/verify'): return try: verify(self.app, self.image) except Exception as exc: self.logger.fatal(str(exc)) raise DockerVerificationFailed()
def _verify_file(self, fname): if not ucr_is_false('appcenter/index/verify'): detached_sig_path = fname + '.gpg' (rc, gpg_error) = gpg_verify(fname, detached_sig_path) if rc: if gpg_error: self.fatal(gpg_error) raise UpdateSignatureVerificationFailed(fname)
def _load_index_json(self, app_cache): index_json_gz_filename = os.path.join(app_cache.get_cache_dir(), '.index.json.gz') if not ucr_is_false('appcenter/index/verify'): detached_sig_path = index_json_gz_filename + '.gpg' (rc, gpg_error) = gpg_verify(index_json_gz_filename, detached_sig_path) if rc: if gpg_error: self.fatal(gpg_error) raise Abort('Signature verification for %s failed' % index_json_gz_filename) with gzip_open(index_json_gz_filename, 'rb') as fgzip: content = fgzip.read() return loads(content)
def to_dict(cls, apps): self = cls() lo, pos = self._get_ldap_connection(args=None, allow_machine_connection=True) hosts = self.get_appcenter_hosts(lo, pos) if ucr_is_false('appcenter/domainwide'): hostname = ucr_get('hostname') hosts = [host for host in hosts if host['name'] == hostname] get = get_action('get') ret = [] app_ldap_objects = search_objects('appcenter/app', lo, pos) for app in apps: if not app: ret.append(None) else: app_dict = get.to_dict(app) app_dict['installations'] = self._get_installations(app, hosts, app_ldap_objects) app_dict['is_installed_anywhere'] = any(inst['version'] for inst in app_dict['installations'].itervalues()) app_dict['fully_loaded'] = True ret.append(app_dict) return ret