def _download_apps(self, app_cache):
filenames = ['index.json.gz']
if not ucr_is_false('appcenter/index/verify'):
filenames.append('index.json.gz.gpg')
self._download_files(app_cache, filenames)
json_apps = self._load_index_json(app_cache)
files_to_download, something_changed_remotely = self._read_index_json(app_cache.get_cache_dir(), json_apps)
num_files_to_be_downloaded = len(files_to_download)
self.log('%d file(s) are new' % num_files_to_be_downloaded)
num_files_threshold = 5
if num_files_to_be_downloaded > num_files_threshold:
files_to_download = self._download_archive(app_cache, files_to_download)
threads = []
max_threads = 10
files_per_thread = max(num_files_threshold, int(ceil(float(len(files_to_download)) / max_threads)))
while files_to_download:
# normally, this should be only one thread as
# _download_archive() is used if many files are to be downloaded
# but if all.tar.gz fails, everything needs to be downloaded
# don't do this at once as this opens 100 connections.
files_to_download_in_thread, files_to_download = files_to_download[:files_per_thread], files_to_download[files_per_thread:]
self.log('Starting to download %d file(s) directly' % len(files_to_download_in_thread))
thread = Thread(target=self._download_directly, args=(app_cache, files_to_download_in_thread,))
thread.start()
threads.append(thread)
time.sleep(0.1) # wait 100 milliseconds so that not all threads start at the same time
for thread in threads:
thread.join()
return something_changed_remotely
def _download_apps(self, app_cache):
filenames = ['index.json.gz']
if not ucr_is_false('appcenter/index/verify'):
filenames.append('index.json.gz.gpg')
filenames.append('all.tar.gpg')
if self._download_files(app_cache, filenames):
appcenter_host = app_cache.get_server()
if appcenter_host.startswith('https'):
appcenter_host = 'http://%s' % appcenter_host[8:]
all_tar_file = os.path.join(app_cache.get_cache_dir(), '.all.tar')
all_tar_url = '%s/meta-inf/%s/all.tar.zsync' % (appcenter_host, app_cache.get_ucs_version())
self.log('Downloading "%s"...' % all_tar_url)
cwd = os.getcwd()
os.chdir(os.path.dirname(all_tar_file))
try:
if self._subprocess(['zsync', all_tar_url, '-q', '-o', all_tar_file]).returncode:
# fallback: download all.tar.gz without zsync. some proxys have difficulties with it, including:
# * Range requests are not supported
# * HTTP requests are altered
self.warn('Downloading the App archive via zsync failed. Falling back to download it directly.')
self.warn('For better performance, try to make zsync work for "%s". The error may be caused by a proxy altering HTTP requests' % all_tar_url)
self._download_files(app_cache, ['all.tar.gz'])
self._uncompress_archive(app_cache, os.path.join(app_cache.get_cache_dir(), '.all.tar.gz'))
finally:
os.chdir(cwd)
self._verify_file(all_tar_file)
self._extract_archive(app_cache)
return True
return False
def verify(self):
if ucr_is_false('appcenter/index/verify'):
return
try:
verify(self.app, self.image)
except Exception as exc:
self.logger.fatal(str(exc))
raise DockerVerificationFailed()
def _verify_file(self, fname):
if not ucr_is_false('appcenter/index/verify'):
detached_sig_path = fname + '.gpg'
(rc, gpg_error) = gpg_verify(fname, detached_sig_path)
if rc:
if gpg_error:
self.fatal(gpg_error)
raise UpdateSignatureVerificationFailed(fname)
def _load_index_json(self, app_cache):
index_json_gz_filename = os.path.join(app_cache.get_cache_dir(), '.index.json.gz')
if not ucr_is_false('appcenter/index/verify'):
detached_sig_path = index_json_gz_filename + '.gpg'
(rc, gpg_error) = gpg_verify(index_json_gz_filename, detached_sig_path)
if rc:
if gpg_error:
self.fatal(gpg_error)
raise Abort('Signature verification for %s failed' % index_json_gz_filename)
with gzip_open(index_json_gz_filename, 'rb') as fgzip:
content = fgzip.read()
return loads(content)
def to_dict(cls, apps):
self = cls()
lo, pos = self._get_ldap_connection(args=None, allow_machine_connection=True)
hosts = self.get_appcenter_hosts(lo, pos)
if ucr_is_false('appcenter/domainwide'):
hostname = ucr_get('hostname')
hosts = [host for host in hosts if host['name'] == hostname]
get = get_action('get')
ret = []
app_ldap_objects = search_objects('appcenter/app', lo, pos)
for app in apps:
if not app:
ret.append(None)
else:
app_dict = get.to_dict(app)
app_dict['installations'] = self._get_installations(app, hosts, app_ldap_objects)
app_dict['is_installed_anywhere'] = any(inst['version'] for inst in app_dict['installations'].itervalues())
app_dict['fully_loaded'] = True
ret.append(app_dict)
return ret
