def __init__(self): super(QObject, self).__init__() self._conda_api = CondaAPI() self._queue = deque() self._threads = [] self._workers = [] self._timer = QTimer() self._chunk_size = 1024 self._timer.setInterval(1000) self._timer.timeout.connect(self._clean)
def __init__(self, load_rc_func=None): """Download API based on requests.""" super(QObject, self).__init__() self._conda_api = CondaAPI() self._queue = deque() self._threads = [] self._workers = [] self._timer = QTimer() self._load_rc_func = load_rc_func self._chunk_size = 1024 self._timer.setInterval(1000) self._timer.timeout.connect(self._clean)
def __init__(self): super(QObject, self).__init__() self._anaconda_client_api = binstar_client.utils.get_server_api( log_level=logging.NOTSET) self._queue = deque() self._threads = [] self._workers = [] self._timer = QTimer() self._conda_api = CondaAPI() self._timer.setInterval(1000) self._timer.timeout.connect(self._clean)
def __init__(self): """Anaconda Manager API process worker.""" super(_ManagerAPI, self).__init__() # API's self._conda_api = CondaAPI() self._client_api = ClientAPI() self._download_api = DownloadAPI(load_rc_func=self._conda_api.load_rc) self._requests_download_api = RequestsDownloadAPI( load_rc_func=self._conda_api.load_rc) self.ROOT_PREFIX = self._conda_api.ROOT_PREFIX # Vars self._checking_repos = None self._data_directory = None self._files_downloaded = None self._repodata_files = None self._valid_repos = None # Expose some methods for convenient access. Methods return a worker self.conda_create = self._conda_api.create self.conda_create_yaml = self._conda_api.create_from_yaml self.conda_clone = self._conda_api.clone_environment self.conda_dependencies = self._conda_api.dependencies self.conda_get_condarc_channels = self._conda_api.get_condarc_channels self.conda_install = self._conda_api.install self.conda_remove = self._conda_api.remove self.conda_terminate = self._conda_api.terminate_all_processes self.conda_config_add = self._conda_api.config_add self.conda_config_remove = self._conda_api.config_remove self.pip_list = self._conda_api.pip_list self.pip_remove = self._conda_api.pip_remove # No workers are returned for these methods self.conda_clear_lock = self._conda_api.clear_lock self.conda_environment_exists = self._conda_api.environment_exists self.conda_get_envs = self._conda_api.get_envs self.conda_linked = self._conda_api.linked self.conda_get_prefix_envname = self._conda_api.get_prefix_envname self.conda_package_version = self._conda_api.package_version self.conda_platform = self._conda_api.get_platform # These download methods return a worker get_api_info = self._requests_download_api.get_api_info is_valid_url = self._requests_download_api.is_valid_api_url is_valid_channel = self._requests_download_api.is_valid_channel terminate = self._requests_download_api.terminate self.download_requests = self._requests_download_api.download self.download_async = self._download_api.download self.download_async_terminate = self._download_api.terminate self.download_is_valid_url = self._requests_download_api.is_valid_url self.download_is_valid_api_url = is_valid_url self.download_get_api_info = lambda: get_api_info(self._client_api. get_api_url()) self.download_is_valid_channel = is_valid_channel self.download_requests_terminate = terminate # These client methods return a worker self.client_store_token = self._client_api.store_token self.client_remove_token = self._client_api.remove_token self.client_login = self._client_api.login self.client_logout = self._client_api.logout self.client_load_repodata = self._client_api.load_repodata self.client_prepare_packages_data = self._client_api.prepare_model_data self.client_user = self._client_api.user self.client_domain = self._client_api.domain self.client_set_domain = self._client_api.set_domain self.client_packages = self._client_api.packages self.client_multi_packages = self._client_api.multi_packages self.client_organizations = self._client_api.organizations self.client_load_token = self._client_api.load_token self.client_get_api_url = self._client_api.get_api_url self.client_set_api_url = self._client_api.set_api_url
class _ManagerAPI(QObject): """Anaconda Manager API process worker.""" sig_repodata_updated = Signal(object) sig_repodata_errored = Signal() def __init__(self): """Anaconda Manager API process worker.""" super(_ManagerAPI, self).__init__() # API's self._conda_api = CondaAPI() self._client_api = ClientAPI() self._download_api = DownloadAPI(load_rc_func=self._conda_api.load_rc) self._requests_download_api = RequestsDownloadAPI( load_rc_func=self._conda_api.load_rc) self.ROOT_PREFIX = self._conda_api.ROOT_PREFIX # Vars self._checking_repos = None self._data_directory = None self._files_downloaded = None self._repodata_files = None self._valid_repos = None # Expose some methods for convenient access. Methods return a worker self.conda_create = self._conda_api.create self.conda_create_yaml = self._conda_api.create_from_yaml self.conda_clone = self._conda_api.clone_environment self.conda_dependencies = self._conda_api.dependencies self.conda_get_condarc_channels = self._conda_api.get_condarc_channels self.conda_install = self._conda_api.install self.conda_remove = self._conda_api.remove self.conda_terminate = self._conda_api.terminate_all_processes self.conda_config_add = self._conda_api.config_add self.conda_config_remove = self._conda_api.config_remove self.pip_list = self._conda_api.pip_list self.pip_remove = self._conda_api.pip_remove # No workers are returned for these methods self.conda_clear_lock = self._conda_api.clear_lock self.conda_environment_exists = self._conda_api.environment_exists self.conda_get_envs = self._conda_api.get_envs self.conda_linked = self._conda_api.linked self.conda_get_prefix_envname = self._conda_api.get_prefix_envname self.conda_package_version = self._conda_api.package_version self.conda_platform = self._conda_api.get_platform # These download methods return a worker get_api_info = self._requests_download_api.get_api_info is_valid_url = self._requests_download_api.is_valid_api_url is_valid_channel = self._requests_download_api.is_valid_channel terminate = self._requests_download_api.terminate self.download_requests = self._requests_download_api.download self.download_async = self._download_api.download self.download_async_terminate = self._download_api.terminate self.download_is_valid_url = self._requests_download_api.is_valid_url self.download_is_valid_api_url = is_valid_url self.download_get_api_info = lambda: get_api_info(self._client_api. get_api_url()) self.download_is_valid_channel = is_valid_channel self.download_requests_terminate = terminate # These client methods return a worker self.client_store_token = self._client_api.store_token self.client_remove_token = self._client_api.remove_token self.client_login = self._client_api.login self.client_logout = self._client_api.logout self.client_load_repodata = self._client_api.load_repodata self.client_prepare_packages_data = self._client_api.prepare_model_data self.client_user = self._client_api.user self.client_domain = self._client_api.domain self.client_set_domain = self._client_api.set_domain self.client_packages = self._client_api.packages self.client_multi_packages = self._client_api.multi_packages self.client_organizations = self._client_api.organizations self.client_load_token = self._client_api.load_token self.client_get_api_url = self._client_api.get_api_url self.client_set_api_url = self._client_api.set_api_url # --- Helper methods # ------------------------------------------------------------------------- def _set_repo_urls_from_channels(self, channels): """ Convert a channel into a normalized repo name including. Channels are assumed in normalized url form. """ repos = [] sys_platform = self._conda_api.get_platform() for channel in channels: url = '{0}/{1}/repodata.json.bz2'.format(channel, sys_platform) repos.append(url) return repos def _check_repos(self, repos): """Check if repodata urls are valid.""" self._checking_repos = [] self._valid_repos = [] for repo in repos: worker = self.download_is_valid_url(repo) worker.sig_finished.connect(self._repos_checked) worker.repo = repo self._checking_repos.append(repo) def _repos_checked(self, worker, output, error): """Callback for _check_repos.""" if worker.repo in self._checking_repos: self._checking_repos.remove(worker.repo) if output: self._valid_repos.append(worker.repo) if len(self._checking_repos) == 0: self._download_repodata(self._valid_repos) def _repo_url_to_path(self, repo): """Convert a `repo` url to a file path for local storage.""" repo = repo.replace('http://', '') repo = repo.replace('https://', '') repo = repo.replace('/', '_') return os.sep.join([self._data_directory, repo]) def _download_repodata(self, checked_repos): """Dowload repodata.""" self._files_downloaded = [] self._repodata_files = [] self.__counter = -1 if checked_repos: for repo in checked_repos: path = self._repo_url_to_path(repo) self._files_downloaded.append(path) self._repodata_files.append(path) worker = self.download_async(repo, path) worker.url = repo worker.path = path worker.sig_finished.connect(self._repodata_downloaded) else: # Empty, maybe there is no internet connection # Load information from conda-meta and save that file path = self._get_repodata_from_meta() self._repodata_files = [path] self._repodata_downloaded() def _get_repodata_from_meta(self): """Generate repodata from local meta files.""" path = os.sep.join([self.ROOT_PREFIX, 'conda-meta']) packages = os.listdir(path) meta_repodata = {} for pkg in packages: if pkg.endswith('.json'): filepath = os.sep.join([path, pkg]) with open(filepath, 'r') as f: data = json.load(f) if 'files' in data: data.pop('files') if 'icondata' in data: data.pop('icondata') name = pkg.replace('.json', '') meta_repodata[name] = data meta_repodata_path = os.sep.join( [self._data_directory, 'offline.json']) repodata = {'info': [], 'packages': meta_repodata} with open(meta_repodata_path, 'w') as f: json.dump(repodata, f, sort_keys=True, indent=4, separators=(',', ': ')) return meta_repodata_path def _repodata_downloaded(self, worker=None, output=None, error=None): """Callback for _download_repodata.""" if worker: self._files_downloaded.remove(worker.path) if worker.path in self._files_downloaded: self._files_downloaded.remove(worker.path) if len(self._files_downloaded) == 0: self.sig_repodata_updated.emit(list(set(self._repodata_files))) # --- Public API # ------------------------------------------------------------------------- def repodata_files(self, channels=None): """ Return the repodata paths based on `channels` and the `data_directory`. There is no check for validity here. """ if channels is None: channels = self.conda_get_condarc_channels() repodata_urls = self._set_repo_urls_from_channels(channels) repopaths = [] for repourl in repodata_urls: fullpath = os.sep.join([self._repo_url_to_path(repourl)]) repopaths.append(fullpath) return repopaths def set_data_directory(self, data_directory): """Set the directory where repodata and metadata are stored.""" self._data_directory = data_directory def update_repodata(self, channels=None): """Update repodata from channels or use condarc channels if None.""" norm_channels = self.conda_get_condarc_channels(channels=channels, normalize=True) repodata_urls = self._set_repo_urls_from_channels(norm_channels) self._check_repos(repodata_urls) def update_metadata(self): """ Update the metadata available for packages in repo.continuum.io. Returns a download worker. """ if self._data_directory is None: raise Exception('Need to call `api.set_data_directory` first.') metadata_url = 'https://repo.continuum.io/pkgs/metadata.json' filepath = os.sep.join([self._data_directory, 'metadata.json']) worker = self.download_requests(metadata_url, filepath) return worker def check_valid_channel(self, channel, conda_url='https://conda.anaconda.org'): """Check if channel is valid.""" if channel.startswith('https://') or channel.startswith('http://'): url = channel else: url = "{0}/{1}".format(conda_url, channel) if url[-1] == '/': url = url[:-1] plat = self.conda_platform() repodata_url = "{0}/{1}/{2}".format(url, plat, 'repodata.json') worker = self.download_is_valid_url(repodata_url) worker.url = url return worker
class _RequestsDownloadAPI(QObject): """Download API based on requests.""" _sig_download_finished = Signal(str, str) _sig_download_progress = Signal(str, str, int, int) def __init__(self, load_rc_func=None): """Download API based on requests.""" super(QObject, self).__init__() self._conda_api = CondaAPI() self._queue = deque() self._threads = [] self._workers = [] self._timer = QTimer() self._load_rc_func = load_rc_func self._chunk_size = 1024 self._timer.setInterval(1000) self._timer.timeout.connect(self._clean) @property def proxy_servers(self): """Return the proxy servers available from the conda rc config file.""" if self._load_rc_func is None: return {} else: return self._load_rc_func().get('proxy_servers', {}) def _clean(self): """Check for inactive workers and remove their references.""" if self._workers: for w in self._workers: if w.is_finished(): self._workers.remove(w) if self._threads: for t in self._threads: if t.isFinished(): self._threads.remove(t) else: self._timer.stop() def _start(self): """Start the next threaded worker in the queue.""" if len(self._queue) == 1: thread = self._queue.popleft() thread.start() self._timer.start() def _create_worker(self, method, *args, **kwargs): """Create a new worker instance.""" thread = QThread() worker = RequestsDownloadWorker(method, args, kwargs) worker.moveToThread(thread) worker.sig_finished.connect(self._start) self._sig_download_finished.connect(worker.sig_download_finished) self._sig_download_progress.connect(worker.sig_download_progress) worker.sig_finished.connect(thread.quit) thread.started.connect(worker.start) self._queue.append(thread) self._threads.append(thread) self._workers.append(worker) self._start() return worker def _download(self, url, path=None, force=False): """Callback for download.""" if path is None: path = url.split('/')[-1] # Make dir if non existent folder = os.path.dirname(os.path.abspath(path)) if not os.path.isdir(folder): os.makedirs(folder) # Start actual download try: r = requests.get(url, stream=True, proxies=self.proxy_servers) except Exception as error: print('ERROR', 'here', error) logger.error(str(error)) # Break if error found! # self._sig_download_finished.emit(url, path) # return path total_size = int(r.headers.get('Content-Length', 0)) # Check if file exists if os.path.isfile(path) and not force: file_size = os.path.getsize(path) # Check if existing file matches size of requested file if file_size == total_size: self._sig_download_finished.emit(url, path) return path # File not found or file size did not match. Download file. progress_size = 0 with open(path, 'wb') as f: for chunk in r.iter_content(chunk_size=self._chunk_size): if chunk: f.write(chunk) progress_size += len(chunk) self._sig_download_progress.emit(url, path, progress_size, total_size) self._sig_download_finished.emit(url, path) return path def _is_valid_url(self, url): """Callback for is_valid_url.""" try: r = requests.head(url, proxies=self.proxy_servers) value = r.status_code in [200] except Exception as error: logger.error(str(error)) value = False return value def _is_valid_channel(self, channel, conda_url='https://conda.anaconda.org'): """Callback for is_valid_channel.""" if channel.startswith('https://') or channel.startswith('http://'): url = channel else: url = "{0}/{1}".format(conda_url, channel) if url[-1] == '/': url = url[:-1] plat = self._conda_api.get_platform() repodata_url = "{0}/{1}/{2}".format(url, plat, 'repodata.json') try: r = requests.head(repodata_url, proxies=self.proxy_servers) value = r.status_code in [200] except Exception as error: logger.error(str(error)) value = False return value def _is_valid_api_url(self, url): """Callback for is_valid_api_url.""" # Check response is a JSON with ok: 1 data = {} try: r = requests.get(url, proxies=self.proxy_servers) content = to_text_string(r.content, encoding='utf-8') data = json.loads(content) except Exception as error: logger.error(str(error)) return data.get('ok', 0) == 1 # --- Public API # ------------------------------------------------------------------------- def download(self, url, path=None, force=False): """Download file given by url and save it to path.""" logger.debug(str((url, path, force))) method = self._download return self._create_worker(method, url, path=path, force=force) def terminate(self): """Terminate all workers and threads.""" for t in self._threads: t.quit() self._thread = [] self._workers = [] def is_valid_url(self, url, non_blocking=True): """Check if url is valid.""" logger.debug(str((url))) if non_blocking: method = self._is_valid_url return self._create_worker(method, url) else: return self._is_valid_url(url) def is_valid_api_url(self, url, non_blocking=True): """Check if anaconda api url is valid.""" logger.debug(str((url))) if non_blocking: method = self._is_valid_api_url return self._create_worker(method, url) else: return self._is_valid_api_url(url=url) def is_valid_channel(self, channel, conda_url='https://conda.anaconda.org', non_blocking=True): """Check if a conda channel is valid.""" logger.debug(str((channel, conda_url))) if non_blocking: method = self._is_valid_channel return self._create_worker(method, channel, conda_url) else: return self._is_valid_channel(channel, conda_url=conda_url) def get_api_info(self, url): """Query anaconda api info.""" data = {} try: r = requests.get(url, proxies=self.proxy_servers) content = to_text_string(r.content, encoding='utf-8') data = json.loads(content) if not data: data['api_url'] = url if 'conda_url' not in data: data['conda_url'] = 'https://conda.anaconda.org' except Exception as error: logger.error(str(error)) return data
class _RequestsDownloadAPI(QObject): """ """ _sig_download_finished = Signal(str, str) _sig_download_progress = Signal(str, str, int, int) def __init__(self): super(QObject, self).__init__() self._conda_api = CondaAPI() self._queue = deque() self._threads = [] self._workers = [] self._timer = QTimer() self._chunk_size = 1024 self._timer.setInterval(1000) self._timer.timeout.connect(self._clean) def _clean(self): """ Periodically check for inactive workers and remove their references. """ if self._workers: for w in self._workers: if w.is_finished(): self._workers.remove(w) if self._threads: for t in self._threads: if t.isFinished(): self._threads.remove(t) else: self._timer.stop() def _start(self): """ """ if len(self._queue) == 1: thread = self._queue.popleft() thread.start() self._timer.start() def _create_worker(self, method, *args, **kwargs): """ """ # FIXME: this might be heavy... thread = QThread() worker = RequestsDownloadWorker(method, args, kwargs) worker.moveToThread(thread) worker.sig_finished.connect(self._start) self._sig_download_finished.connect(worker.sig_download_finished) self._sig_download_progress.connect(worker.sig_download_progress) worker.sig_finished.connect(thread.quit) thread.started.connect(worker.start) self._queue.append(thread) self._threads.append(thread) self._workers.append(worker) self._start() return worker def _download(self, url, path=None, force=False): """ """ if path is None: path = url.split('/')[-1] # Make dir if non existent folder = os.path.dirname(os.path.abspath(path)) if not os.path.isdir(folder): os.makedirs(folder) # Start actual download try: r = requests.get(url, stream=True) except Exception as error: logger.error(str(error)) # Break if error found! # self._sig_download_finished.emit(url, path) # return path total_size = int(r.headers.get('Content-Length', 0)) # Check if file exists if os.path.isfile(path) and not force: file_size = os.path.getsize(path) # Check if existing file matches size of requested file if file_size == total_size: self._sig_download_finished.emit(url, path) return path # File not found or file size did not match. Download file. progress_size = 0 with open(path, 'wb') as f: for chunk in r.iter_content(chunk_size=self._chunk_size): if chunk: f.write(chunk) progress_size += len(chunk) self._sig_download_progress.emit(url, path, progress_size, total_size) self._sig_download_finished.emit(url, path) return path def _is_valid_url(self, url): try: r = requests.head(url) value = r.status_code in [200] except Exception as error: logger.error(str(error)) value = False return value def _is_valid_channel(self, channel, conda_url='https://conda.anaconda.org'): """ """ if channel.startswith('https://') or channel.startswith('http://'): url = channel else: url = "{0}/{1}".format(conda_url, channel) if url[-1] == '/': url = url[:-1] plat = self._conda_api.get_platform() repodata_url = "{0}/{1}/{2}".format(url, plat, 'repodata.json') try: r = requests.head(repodata_url) value = r.status_code in [200] except Exception as error: logger.error(str(error)) value = False return value def _is_valid_api_url(self, url): """ """ # Check response is a JSON with ok: 1 data = {} try: r = requests.get(url) content = to_text_string(r.content, encoding='utf-8') data = json.loads(content) except Exception as error: logger.error(str(error)) return data.get('ok', 0) == 1 def download(self, url, path=None, force=False): logger.debug(str((url, path, force))) method = self._download return self._create_worker(method, url, path=path, force=force) def terminate(self): for t in self._threads: t.quit() self._thread = [] self._workers = [] def is_valid_url(self, url, non_blocking=True): logger.debug(str((url))) if non_blocking: method = self._is_valid_url return self._create_worker(method, url) else: return self._is_valid_url(url) def is_valid_api_url(self, url, non_blocking=True): logger.debug(str((url))) if non_blocking: method = self._is_valid_api_url return self._create_worker(method, url) else: return self._is_valid_api_url(url=url) def is_valid_channel(self, channel, conda_url='https://conda.anaconda.org', non_blocking=True): logger.debug(str((channel, conda_url))) if non_blocking: method = self._is_valid_channel return self._create_worker(method, channel, conda_url) else: return self._is_valid_channel(channel, conda_url=conda_url)
def __init__(self): """Anaconda Manager API process worker.""" super(_ManagerAPI, self).__init__() # API's self._conda_api = CondaAPI() self._client_api = ClientAPI() self._download_api = DownloadAPI(load_rc_func=self._conda_api.load_rc) self._requests_download_api = RequestsDownloadAPI( load_rc_func=self._conda_api.load_rc) self.ROOT_PREFIX = self._conda_api.ROOT_PREFIX # Vars self._checking_repos = None self._data_directory = None self._files_downloaded = None self._repodata_files = None self._valid_repos = None # Expose some methods for convenient access. Methods return a worker self.conda_create = self._conda_api.create self.conda_create_yaml = self._conda_api.create_from_yaml self.conda_clone = self._conda_api.clone_environment self.conda_dependencies = self._conda_api.dependencies self.conda_get_condarc_channels = self._conda_api.get_condarc_channels self.conda_install = self._conda_api.install self.conda_remove = self._conda_api.remove self.conda_terminate = self._conda_api.terminate_all_processes self.conda_config_add = self._conda_api.config_add self.conda_config_remove = self._conda_api.config_remove self.pip_list = self._conda_api.pip_list self.pip_remove = self._conda_api.pip_remove # No workers are returned for these methods self.conda_clear_lock = self._conda_api.clear_lock self.conda_environment_exists = self._conda_api.environment_exists self.conda_get_envs = self._conda_api.get_envs self.conda_linked = self._conda_api.linked self.conda_get_prefix_envname = self._conda_api.get_prefix_envname self.conda_package_version = self._conda_api.package_version self.conda_platform = self._conda_api.get_platform # These download methods return a worker get_api_info = self._requests_download_api.get_api_info is_valid_url = self._requests_download_api.is_valid_api_url is_valid_channel = self._requests_download_api.is_valid_channel terminate = self._requests_download_api.terminate self.download_requests = self._requests_download_api.download self.download_async = self._download_api.download self.download_async_terminate = self._download_api.terminate self.download_is_valid_url = self._requests_download_api.is_valid_url self.download_is_valid_api_url = is_valid_url self.download_get_api_info = lambda: get_api_info( self._client_api.get_api_url()) self.download_is_valid_channel = is_valid_channel self.download_requests_terminate = terminate # These client methods return a worker self.client_store_token = self._client_api.store_token self.client_remove_token = self._client_api.remove_token self.client_login = self._client_api.login self.client_logout = self._client_api.logout self.client_load_repodata = self._client_api.load_repodata self.client_prepare_packages_data = self._client_api.prepare_model_data self.client_user = self._client_api.user self.client_domain = self._client_api.domain self.client_set_domain = self._client_api.set_domain self.client_packages = self._client_api.packages self.client_multi_packages = self._client_api.multi_packages self.client_organizations = self._client_api.organizations self.client_load_token = self._client_api.load_token self.client_get_api_url = self._client_api.get_api_url self.client_set_api_url = self._client_api.set_api_url
class _ManagerAPI(QObject): """Anaconda Manager API process worker.""" sig_repodata_updated = Signal(object) sig_repodata_errored = Signal() def __init__(self): """Anaconda Manager API process worker.""" super(_ManagerAPI, self).__init__() # API's self._conda_api = CondaAPI() self._client_api = ClientAPI() self._download_api = DownloadAPI(load_rc_func=self._conda_api.load_rc) self._requests_download_api = RequestsDownloadAPI( load_rc_func=self._conda_api.load_rc) self.ROOT_PREFIX = self._conda_api.ROOT_PREFIX # Vars self._checking_repos = None self._data_directory = None self._files_downloaded = None self._repodata_files = None self._valid_repos = None # Expose some methods for convenient access. Methods return a worker self.conda_create = self._conda_api.create self.conda_create_yaml = self._conda_api.create_from_yaml self.conda_clone = self._conda_api.clone_environment self.conda_dependencies = self._conda_api.dependencies self.conda_get_condarc_channels = self._conda_api.get_condarc_channels self.conda_install = self._conda_api.install self.conda_remove = self._conda_api.remove self.conda_terminate = self._conda_api.terminate_all_processes self.conda_config_add = self._conda_api.config_add self.conda_config_remove = self._conda_api.config_remove self.pip_list = self._conda_api.pip_list self.pip_remove = self._conda_api.pip_remove # No workers are returned for these methods self.conda_clear_lock = self._conda_api.clear_lock self.conda_environment_exists = self._conda_api.environment_exists self.conda_get_envs = self._conda_api.get_envs self.conda_linked = self._conda_api.linked self.conda_get_prefix_envname = self._conda_api.get_prefix_envname self.conda_package_version = self._conda_api.package_version self.conda_platform = self._conda_api.get_platform # These download methods return a worker get_api_info = self._requests_download_api.get_api_info is_valid_url = self._requests_download_api.is_valid_api_url is_valid_channel = self._requests_download_api.is_valid_channel terminate = self._requests_download_api.terminate self.download_requests = self._requests_download_api.download self.download_async = self._download_api.download self.download_async_terminate = self._download_api.terminate self.download_is_valid_url = self._requests_download_api.is_valid_url self.download_is_valid_api_url = is_valid_url self.download_get_api_info = lambda: get_api_info( self._client_api.get_api_url()) self.download_is_valid_channel = is_valid_channel self.download_requests_terminate = terminate # These client methods return a worker self.client_store_token = self._client_api.store_token self.client_remove_token = self._client_api.remove_token self.client_login = self._client_api.login self.client_logout = self._client_api.logout self.client_load_repodata = self._client_api.load_repodata self.client_prepare_packages_data = self._client_api.prepare_model_data self.client_user = self._client_api.user self.client_domain = self._client_api.domain self.client_set_domain = self._client_api.set_domain self.client_packages = self._client_api.packages self.client_multi_packages = self._client_api.multi_packages self.client_organizations = self._client_api.organizations self.client_load_token = self._client_api.load_token self.client_get_api_url = self._client_api.get_api_url self.client_set_api_url = self._client_api.set_api_url # --- Helper methods # ------------------------------------------------------------------------- def _set_repo_urls_from_channels(self, channels): """ Convert a channel into a normalized repo name including. Channels are assumed in normalized url form. """ repos = [] sys_platform = self._conda_api.get_platform() for channel in channels: url = '{0}/{1}/repodata.json.bz2'.format(channel, sys_platform) repos.append(url) return repos def _check_repos(self, repos): """Check if repodata urls are valid.""" self._checking_repos = [] self._valid_repos = [] for repo in repos: worker = self.download_is_valid_url(repo) worker.sig_finished.connect(self._repos_checked) worker.repo = repo self._checking_repos.append(repo) def _repos_checked(self, worker, output, error): """Callback for _check_repos.""" if worker.repo in self._checking_repos: self._checking_repos.remove(worker.repo) if output: self._valid_repos.append(worker.repo) if len(self._checking_repos) == 0: self._download_repodata(self._valid_repos) def _repo_url_to_path(self, repo): """Convert a `repo` url to a file path for local storage.""" repo = repo.replace('http://', '') repo = repo.replace('https://', '') repo = repo.replace('/', '_') return os.sep.join([self._data_directory, repo]) def _download_repodata(self, checked_repos): """Dowload repodata.""" self._files_downloaded = [] self._repodata_files = [] self.__counter = -1 if checked_repos: for repo in checked_repos: path = self._repo_url_to_path(repo) self._files_downloaded.append(path) self._repodata_files.append(path) worker = self.download_async(repo, path) worker.url = repo worker.path = path worker.sig_finished.connect(self._repodata_downloaded) else: # Empty, maybe there is no internet connection # Load information from conda-meta and save that file path = self._get_repodata_from_meta() self._repodata_files = [path] self._repodata_downloaded() def _get_repodata_from_meta(self): """Generate repodata from local meta files.""" path = os.sep.join([self.ROOT_PREFIX, 'conda-meta']) packages = os.listdir(path) meta_repodata = {} for pkg in packages: if pkg.endswith('.json'): filepath = os.sep.join([path, pkg]) with open(filepath, 'r') as f: data = json.load(f) if 'files' in data: data.pop('files') if 'icondata' in data: data.pop('icondata') name = pkg.replace('.json', '') meta_repodata[name] = data meta_repodata_path = os.sep.join([self._data_directory, 'offline.json']) repodata = {'info': [], 'packages': meta_repodata} with open(meta_repodata_path, 'w') as f: json.dump(repodata, f, sort_keys=True, indent=4, separators=(',', ': ')) return meta_repodata_path def _repodata_downloaded(self, worker=None, output=None, error=None): """Callback for _download_repodata.""" if worker: self._files_downloaded.remove(worker.path) if worker.path in self._files_downloaded: self._files_downloaded.remove(worker.path) if len(self._files_downloaded) == 0: self.sig_repodata_updated.emit(list(set(self._repodata_files))) # --- Public API # ------------------------------------------------------------------------- def repodata_files(self, channels=None): """ Return the repodata paths based on `channels` and the `data_directory`. There is no check for validity here. """ if channels is None: channels = self.conda_get_condarc_channels() repodata_urls = self._set_repo_urls_from_channels(channels) repopaths = [] for repourl in repodata_urls: fullpath = os.sep.join([self._repo_url_to_path(repourl)]) repopaths.append(fullpath) return repopaths def set_data_directory(self, data_directory): """Set the directory where repodata and metadata are stored.""" self._data_directory = data_directory def update_repodata(self, channels=None): """Update repodata from channels or use condarc channels if None.""" norm_channels = self.conda_get_condarc_channels(channels=channels, normalize=True) repodata_urls = self._set_repo_urls_from_channels(norm_channels) self._check_repos(repodata_urls) def update_metadata(self): """ Update the metadata available for packages in repo.continuum.io. Returns a download worker. """ if self._data_directory is None: raise Exception('Need to call `api.set_data_directory` first.') metadata_url = 'https://repo.continuum.io/pkgs/metadata.json' filepath = os.sep.join([self._data_directory, 'metadata.json']) worker = self.download_requests(metadata_url, filepath) return worker def check_valid_channel(self, channel, conda_url='https://conda.anaconda.org'): """Check if channel is valid.""" if channel.startswith('https://') or channel.startswith('http://'): url = channel else: url = "{0}/{1}".format(conda_url, channel) if url[-1] == '/': url = url[:-1] plat = self.conda_platform() repodata_url = "{0}/{1}/{2}".format(url, plat, 'repodata.json') worker = self.download_is_valid_url(repodata_url) worker.url = url return worker