class _DownloadAPI(QObject):
"""Download API based on requests."""
_sig_download_finished = Signal(str, str)
_sig_download_progress = Signal(str, str, int, int)
_sig_partial = Signal(object)
MAX_THREADS = 20
DEFAULT_TIMEOUT = 5 # seconds
def __init__(self, config=None):
"""Download API based on requests."""
super(QObject, self).__init__()
self._conda_api = CondaAPI()
self._client_api = ClientAPI()
self._config = config
self._queue = deque()
self._queue_workers = deque()
self._threads = []
self._workers = []
self._timer = QTimer()
self._timer_worker_delete = QTimer()
self._running_threads = 0
self._bag_collector = deque() # Keeps references to old workers
self._chunk_size = 1024
self._timer.setInterval(333)
self._timer.timeout.connect(self._start)
self._timer_worker_delete.setInterval(5000)
self._timer_worker_delete.timeout.connect(self._clean_workers)
def _clean_workers(self):
"""Delete periodically workers in workers bag."""
while self._bag_collector:
self._bag_collector.popleft()
self._timer_worker_delete.stop()
def _get_verify_ssl(self, verify, set_conda_ssl=True):
"""Get verify ssl."""
if verify is None:
verify_value = self._client_api.get_ssl(
set_conda_ssl=set_conda_ssl, )
else:
verify_value = verify
return verify_value
def _is_internet_available(self):
"""Check initernet availability."""
if self._config:
config_value = self._config.get('main', 'offline_mode')
else:
config_value = False
if config_value:
connectivity = False
else:
connectivity = True # is_internet_available()
return connectivity
@property
def proxy_servers(self):
"""Return the proxy servers available from the conda rc config file."""
return self._conda_api.load_proxy_config()
def _start(self):
"""Start threads and check for inactive workers."""
if self._queue_workers and self._running_threads < self.MAX_THREADS:
# print('Queue: {0} Running: {1} Workers: {2} '
# 'Threads: {3}'.format(len(self._queue_workers),
# self._running_threads,
# len(self._workers),
# len(self._threads)))
self._running_threads += 1
thread = QThread()
worker = self._queue_workers.popleft()
worker.moveToThread(thread)
worker.sig_finished.connect(thread.quit)
thread.started.connect(worker.start)
thread.start()
self._threads.append(thread)
if self._workers:
for w in self._workers:
if w.is_finished():
self._bag_collector.append(w)
self._workers.remove(w)
if self._threads:
for t in self._threads:
if t.isFinished():
self._threads.remove(t)
self._running_threads -= 1
if len(self._threads) == 0 and len(self._workers) == 0:
self._timer.stop()
self._timer_worker_delete.start()
def _create_worker(self, method, *args, **kwargs):
"""Create a new worker instance."""
worker = DownloadWorker(method, args, kwargs)
self._workers.append(worker)
self._queue_workers.append(worker)
self._sig_download_finished.connect(worker.sig_download_finished)
self._sig_download_progress.connect(worker.sig_download_progress)
self._sig_partial.connect(worker._handle_partial)
self._timer.start()
return worker
def _download(
self,
url,
path=None,
force=False,
verify=None,
chunked=True,
):
"""Callback for download."""
verify_value = self._get_verify_ssl(verify, set_conda_ssl=False)
if path is None:
path = url.split('/')[-1]
# Make dir if non existent
folder = os.path.dirname(os.path.abspath(path))
if not os.path.isdir(folder):
os.makedirs(folder)
# Get headers
if self._is_internet_available():
try:
r = requests.head(
url,
proxies=self.proxy_servers,
verify=verify_value,
timeout=self.DEFAULT_TIMEOUT,
)
status_code = r.status_code
except Exception as error:
status_code = -1
logger.error(str(error))
logger.debug('Status code {0} - url'.format(status_code, url))
if status_code != 200:
logger.error('Invalid url {0}'.format(url))
return path
total_size = int(r.headers.get('Content-Length', 0))
# Check if file exists
if os.path.isfile(path) and not force:
file_size = os.path.getsize(path)
else:
file_size = -1
# print(path, total_size, file_size)
# Check if existing file matches size of requested file
if file_size == total_size:
self._sig_download_finished.emit(url, path)
return path
else:
try:
r = requests.get(
url,
stream=chunked,
proxies=self.proxy_servers,
verify=verify_value,
timeout=self.DEFAULT_TIMEOUT,
)
status_code = r.status_code
except Exception as error:
status_code = -1
logger.error(str(error))
# File not found or file size did not match. Download file.
progress_size = 0
bytes_stream = QBuffer() # BytesIO was segfaulting for big files
bytes_stream.open(QBuffer.ReadWrite)
# For some chunked content the app segfaults (with big files)
# so now chunked is a kwarg for this method
if chunked:
for chunk in r.iter_content(chunk_size=self._chunk_size):
# print(url, progress_size, total_size)
if chunk:
bytes_stream.write(chunk)
progress_size += len(chunk)
self._sig_download_progress.emit(
url,
path,
progress_size,
total_size,
)
self._sig_partial.emit({
'url': url,
'path': path,
'progress_size': progress_size,
'total_size': total_size,
})
else:
bytes_stream.write(r.content)
bytes_stream.seek(0)
data = bytes_stream.data()
with open(path, 'wb') as f:
f.write(data)
bytes_stream.close()
self._sig_download_finished.emit(url, path)
return path
def _is_valid_url(self, url, verify=None):
"""Callback for is_valid_url."""
verify_value = self._get_verify_ssl(verify)
if self._is_internet_available():
try:
r = requests.head(
url,
proxies=self.proxy_servers,
verify=verify_value,
timeout=self.DEFAULT_TIMEOUT,
)
value = r.status_code in [200]
except Exception as error:
logger.error(str(error))
value = False
return value
def _is_valid_channel(
self,
channel,
conda_url='https://conda.anaconda.org',
verify=None,
):
"""Callback for is_valid_channel."""
verify_value = self._get_verify_ssl(verify)
if channel.startswith('https://') or channel.startswith('http://'):
url = channel
else:
url = "{0}/{1}".format(conda_url, channel)
if url[-1] == '/':
url = url[:-1]
plat = self._conda_api.get_platform()
repodata_url = "{0}/{1}/{2}".format(url, plat, 'repodata.json')
if self._is_internet_available():
try:
r = requests.head(
repodata_url,
proxies=self.proxy_servers,
verify=verify_value,
timeout=self.DEFAULT_TIMEOUT,
)
value = r.status_code in [200]
except Exception as error:
logger.error(str(error))
value = False
return value
def _is_valid_api_url(self, url, verify=None):
"""Callback for is_valid_api_url."""
verify_value = self._get_verify_ssl(verify)
# Check response is a JSON with ok: 1
data = {}
if verify is None:
verify_value = self._client_api.get_ssl()
else:
verify_value = verify
if self._is_internet_available():
try:
r = requests.get(
url,
proxies=self.proxy_servers,
verify=verify_value,
timeout=self.DEFAULT_TIMEOUT,
)
content = to_text_string(r.content, encoding='utf-8')
data = json.loads(content)
except Exception as error:
logger.error(str(error))
return data.get('ok', 0) == 1
def _get_url(self, url, as_json=False, verify=None):
"""Callback for url checking."""
data = {}
verify_value = self._get_verify_ssl(verify)
if self._is_internet_available():
try:
# See: https://github.com/ContinuumIO/navigator/issues/1485
session = requests.Session()
retry = Retry(connect=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
r = session.get(
url,
proxies=self.proxy_servers,
verify=verify_value,
timeout=self.DEFAULT_TIMEOUT,
)
data = to_text_string(r.content, encoding='utf-8')
if as_json:
data = json.loads(data)
except Exception as error:
logger.error(str(error))
return data
def _get_api_info(self, url, verify=None):
"""Callback."""
verify_value = self._get_verify_ssl(verify)
data = {
"api_url": url,
"api_docs_url": "https://api.anaconda.org/docs",
"conda_url": "https://conda.anaconda.org/",
"main_url": "https://anaconda.org/",
"pypi_url": "https://pypi.anaconda.org/",
"swagger_url": "https://api.anaconda.org/swagger.json",
}
if self._is_internet_available():
try:
r = requests.get(
url,
proxies=self.proxy_servers,
verify=verify_value,
timeout=self.DEFAULT_TIMEOUT,
)
content = to_text_string(r.content, encoding='utf-8')
new_data = json.loads(content)
data['conda_url'] = new_data.get('conda_url',
data['conda_url'])
except Exception as error:
logger.error(str(error))
return data
# --- Public API
# -------------------------------------------------------------------------
def download(self, url, path=None, force=False, verify=None, chunked=True):
"""Download file given by url and save it to path."""
logger.debug(str((url, path, force)))
method = self._download
return self._create_worker(
method,
url,
path=path,
force=force,
verify=verify,
chunked=chunked,
)
def terminate(self):
"""Terminate all workers and threads."""
for t in self._threads:
t.quit()
self._thread = []
self._workers = []
def is_valid_url(self, url, non_blocking=True):
"""Check if url is valid."""
logger.debug(str((url)))
if non_blocking:
method = self._is_valid_url
return self._create_worker(method, url)
else:
return self._is_valid_url(url)
def is_valid_api_url(self, url, non_blocking=True, verify=None):
"""Check if anaconda api url is valid."""
logger.debug(str((url)))
if non_blocking:
method = self._is_valid_api_url
return self._create_worker(method, url, verify=verify)
else:
return self._is_valid_api_url(url=url, verify=verify)
def is_valid_channel(
self,
channel,
conda_url='https://conda.anaconda.org',
non_blocking=True,
):
"""Check if a conda channel is valid."""
logger.debug(str((channel, conda_url)))
if non_blocking:
method = self._is_valid_channel
return self._create_worker(method, channel, conda_url)
else:
return self._is_valid_channel(channel, conda_url=conda_url)
def get_url(self, url, as_json=False, verify=None, non_blocking=True):
"""Get url content."""
logger.debug(str(url))
if non_blocking:
method = self._get_url
return self._create_worker(method,
url,
as_json=as_json,
verify=verify)
else:
return self._get_url(url, as_json=as_json, verify=verify)
def get_api_info(self, url, non_blocking=True):
"""Query anaconda api info."""
logger.debug(str((url, non_blocking)))
if non_blocking:
method = self._get_api_info
return self._create_worker(method, url)
else:
return self._get_api_info(url)
def package_version(self, pkg, name='root'):
api = CondaAPI()
return api.package_version(name=name, pkg=pkg, build=True)
class _ClientAPI(QObject):
"""Anaconda Client API wrapper."""
DEFAULT_TIMEOUT = 6
def __init__(self):
"""Anaconda Client API wrapper."""
super(QObject, self).__init__()
self._anaconda_client_api = binstar_client.utils.get_server_api(
log_level=logging.NOTSET)
self._queue = deque()
self._threads = []
self._workers = []
self._timer = QTimer()
self._conda_api = CondaAPI()
self.config = CONF
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
def _clean(self):
"""Check for inactive workers and remove their references."""
if self._workers:
for w in self._workers:
if w.is_finished():
self._workers.remove(w)
if self._threads:
for t in self._threads:
if t.isFinished():
self._threads.remove(t)
else:
self._timer.stop()
def _start(self):
"""Take avalaible worker from the queue and start it."""
if len(self._queue) == 1:
thread = self._queue.popleft()
thread.start()
self._timer.start()
def _create_worker(self, method, *args, **kwargs):
"""Create a worker for this client to be run in a separate thread."""
# FIXME: this might be heavy...
thread = QThread()
worker = ClientWorker(method, args, kwargs)
worker.moveToThread(thread)
worker.sig_finished.connect(self._start)
worker.sig_finished.connect(thread.quit)
thread.started.connect(worker.start)
self._queue.append(thread)
self._threads.append(thread)
self._workers.append(worker)
self._start()
return worker
# --- Callbacks
# -------------------------------------------------------------------------
@staticmethod
def _load_repodata(repodata, metadata=None, python_version=None):
"""
Load all the available package information.
See load_repadata for full documentation.
"""
metadata = metadata if metadata else {}
# python_version = '.'.join(python_version.split('.')[:2])
all_packages = {}
for channel_url, data in repodata.items():
packages = data.get('packages', {})
for canonical_name in packages:
data = packages[canonical_name]
# Do not filter based on python version
# if (python_version and not is_dependency_met(
# data['depends'], python_version, 'python')):
# continue
name, version, b = tuple(canonical_name.rsplit('-', 2))
if name not in all_packages:
all_packages[name] = {
'versions': set(),
'size': {},
'type': {},
'app_entry': {},
'app_type': {},
}
elif name in metadata:
temp_data = all_packages[name]
temp_data['home'] = metadata[name].get('home', '')
temp_data['license'] = metadata[name].get('license', '')
temp_data['summary'] = metadata[name].get('summary', '')
temp_data['latest_version'] = metadata[name].get('version')
all_packages[name] = temp_data
all_packages[name]['versions'].add(version)
all_packages[name]['size'][version] = data.get('size', '')
# Only the latest builds will have the correct metadata for
# apps, so only store apps that have the app metadata
if data.get('type'):
all_packages[name]['type'][version] = data.get('type')
all_packages[name]['app_entry'][version] = data.get(
'app_entry')
all_packages[name]['app_type'][version] = data.get(
'app_type')
all_apps = {}
for name in all_packages:
versions = sort_versions(list(all_packages[name]['versions']))
all_packages[name]['versions'] = versions[:]
for version in versions:
has_type = all_packages[name].get('type')
# Has type in this case implies being an app
if has_type:
all_apps[name] = all_packages[name].copy()
# Remove all versions that are not apps!
versions = all_apps[name]['versions'][:]
types = all_apps[name]['type']
app_versions = [v for v in versions if v in types]
all_apps[name]['versions'] = app_versions
return all_packages, all_apps
@staticmethod
def _prepare_model_data(packages, linked, pip=None):
"""Prepare model data for the packages table model."""
pip = pip if pip else []
data = []
linked_packages = {}
for canonical_name in linked:
name, version, b = tuple(canonical_name.rsplit('-', 2))
linked_packages[name] = {'version': version}
pip_packages = {}
for canonical_name in pip:
name, version, b = tuple(canonical_name.rsplit('-', 2))
pip_packages[name] = {'version': version}
packages_names = sorted(
list(
set(
list(linked_packages.keys()) + list(pip_packages.keys()) +
list(packages.keys())), ))
for name in packages_names:
p_data = packages.get(name)
summary = p_data.get('summary', '') if p_data else ''
url = p_data.get('home', '') if p_data else ''
license_ = p_data.get('license', '') if p_data else ''
versions = p_data.get('versions', '') if p_data else []
version = p_data.get('latest_version', '') if p_data else ''
if name in pip_packages:
type_ = C.PIP_PACKAGE
version = pip_packages[name].get('version', '')
status = C.INSTALLED
elif name in linked_packages:
type_ = C.CONDA_PACKAGE
version = linked_packages[name].get('version', '')
status = C.INSTALLED
if version in versions:
vers = versions
upgradable = not version == vers[-1] and len(vers) != 1
downgradable = not version == vers[0] and len(vers) != 1
if upgradable and downgradable:
status = C.MIXGRADABLE
elif upgradable:
status = C.UPGRADABLE
elif downgradable:
status = C.DOWNGRADABLE
else:
type_ = C.CONDA_PACKAGE
status = C.NOT_INSTALLED
if version == '' and len(versions) != 0:
version = versions[-1]
row = {
C.COL_ACTION: C.ACTION_NONE,
C.COL_PACKAGE_TYPE: type_,
C.COL_NAME: name,
C.COL_DESCRIPTION: summary.capitalize(),
C.COL_VERSION: version,
C.COL_STATUS: status,
C.COL_URL: url,
C.COL_LICENSE: license_,
C.COL_ACTION_VERSION: None,
}
data.append(row)
return data
def _get_user_licenses(self, products=None):
"""Get user trial/paid licenses from anaconda.org."""
license_data = []
try:
res = self._anaconda_client_api.user_licenses()
license_data = res.get('data', [])
# This should be returning a dict or list not a json string!
if is_text_string(license_data):
license_data = json.loads(license_data)
except Exception:
time.sleep(0.3)
return license_data
# --- Public API
# -------------------------------------------------------------------------
def token(self):
"""Return the current token registered with authenticate."""
return self._anaconda_client_api.token
def load_token(self):
"""Load current authenticated token."""
token = None
try:
token = binstar_client.utils.load_token(self.get_api_url())
except Exception:
pass
return token
def _login(self, username, password, application, application_url):
"""Login callback."""
new_token = self._anaconda_client_api.authenticate(
username, password, application, application_url)
args = Args()
args.site = None
args.token = new_token
binstar_client.utils.store_token(new_token, args)
return new_token
def login(self, username, password, application, application_url):
"""Login to anaconda server."""
logger.debug(str((username, application, application_url)))
method = self._login
return self._create_worker(method, username, password, application,
application_url)
def logout(self):
"""
Logout from anaconda.org.
This method removes the authentication and removes the token.
"""
error = None
args = Args()
args.site = None
args.token = self.token
binstar_client.utils.remove_token(args)
if self.token:
try:
self._anaconda_client_api.remove_authentication()
except binstar_client.errors.Unauthorized as e:
error = e
logger.debug("The token that you are trying to remove may "
"not be valid {}".format(e))
except Exception as e:
error = e
logger.debug("The certificate might be invalid. {}".format(e))
logger.info("logout successful")
return error
def load_repodata(self, repodata, metadata=None, python_version=None):
"""
Load all the available packages information for downloaded repodata.
For downloaded repodata files (repo.continuum.io), additional
data provided (anaconda cloud), and additional metadata and merge into
a single set of packages and apps.
If python_version is not none, exclude all package/versions which
require an incompatible version of python.
Parameters
----------
repodata: dict of dicts
Data loaded from the conda cache directories.
metadata: dict
Metadata info form different sources. For now only from
repo.continuum.
python_version: str
Python version used in preprocessing.
"""
logger.debug('')
method = self._load_repodata
return self._create_worker(
method,
repodata,
metadata=metadata,
python_version=python_version,
)
def prepare_model_data(self, packages, linked, pip=None):
"""Prepare downloaded package info along with pip pacakges info."""
logger.debug('')
method = self._prepare_model_data
return self._create_worker(
method,
packages,
linked,
pip=pip,
)
def set_domain(self, domain='https://api.anaconda.org'):
"""Reset current api domain."""
logger.debug('Setting domain {}'.format(domain))
config = binstar_client.utils.get_config()
config['url'] = domain
try:
binstar_client.utils.set_config(config)
except binstar_client.errors.BinstarError:
logger.error('Could not write anaconda client configuation')
traceback = format_exc()
msg_box = MessageBoxError(
title='Anaconda Client configuration error',
text='Anaconda Client domain could not be updated.<br><br>'
'This may result in Navigator not working properly.<br>',
error='<pre>' + traceback + '</pre>',
report=False,
learn_more=None,
)
msg_box.exec_()
self._anaconda_client_api = binstar_client.utils.get_server_api(
token=None,
log_level=logging.NOTSET,
)
def user(self):
"""Return current logged user information."""
return self.organizations(login=None)
def domain(self):
"""Return current domain."""
return self._anaconda_client_api.domain
def packages(self,
login=None,
platform=None,
package_type=None,
type_=None,
access=None):
"""Return all the available packages for a given user.
Parameters
----------
type_: Optional[str]
Only find packages that have this conda `type`, (i.e. 'app').
access : Optional[str]
Only find packages that have this access level (e.g. 'private',
'authenticated', 'public').
"""
logger.debug('')
method = self._anaconda_client_api.user_packages
return self._create_worker(
method,
login=login,
platform=platform,
package_type=package_type,
type_=type_,
access=access,
)
def organizations(self, login):
"""List all the organizations a user has access to."""
try:
user = self._anaconda_client_api.user(login=login)
except Exception:
user = {}
return user
@staticmethod
def get_api_url():
"""Get the anaconda client url configuration."""
config_data = binstar_client.utils.get_config()
return config_data.get('url', 'https://api.anaconda.org')
@staticmethod
def set_api_url(url):
"""Set the anaconda client url configuration."""
config_data = binstar_client.utils.get_config()
config_data['url'] = url
try:
binstar_client.utils.set_config(config_data)
except Exception as e:
logger.error('Could not write anaconda client configuration')
msg_box = MessageBoxError(
title='Anaconda Client configuration error',
text='Anaconda Client configuration could not be updated.<br>'
'This may result in Navigator not working properly.<br>',
error=e,
report=False,
learn_more=None,
)
msg_box.exec_()
def get_ssl(self):
"""
Get the anaconda client url configuration and set conda accordingly.
"""
config_data = binstar_client.utils.get_config()
value = config_data.get('verify_ssl', True)
# ssl_verify = self._conda_api.config_get('ssl_verify').communicate()
# if ssl_verify != value: # FIXME: Conda rstricted acces to the key
# self._conda_api.config_set('ssl_verify', value).communicate()
self._conda_api.config_set('ssl_verify', value).communicate()
return value
def set_ssl(self, value):
"""Set the anaconda client url configuration."""
config_data = binstar_client.utils.get_config()
config_data['verify_ssl'] = value
try:
binstar_client.utils.set_config(config_data)
self._conda_api.config_set('ssl_verify', value).communicate()
except Exception as e:
logger.error('Could not write anaconda client configuration')
msg_box = MessageBoxError(
title='Anaconda Client configuration error',
text='Anaconda Client configuration could not be updated.<br>'
'This may result in Navigator not working properly.<br>',
error=e,
report=False,
learn_more=None,
)
msg_box.exec_()
def get_user_licenses(self, products=None):
"""Get user trial/paid licenses from anaconda.org."""
logger.debug(str((products)))
method = self._get_user_licenses
return self._create_worker(method, products=products)
def _get_api_info(self, url, proxy_servers=None, verify=True):
"""Callback."""
proxy_servers = proxy_servers or {}
data = {
"api_url": url,
"api_docs_url": "https://api.anaconda.org/docs",
"brand": DEFAULT_BRAND,
"conda_url": "https://conda.anaconda.org",
"main_url": "https://anaconda.org",
"pypi_url": "https://pypi.anaconda.org",
"swagger_url": "https://api.anaconda.org/swagger.json",
}
try:
r = requests.get(
url,
proxies=proxy_servers,
verify=verify,
timeout=self.DEFAULT_TIMEOUT,
)
content = to_text_string(r.content, encoding='utf-8')
new_data = json.loads(content)
# Enforce no trailing slash
for key, value in new_data.items():
if is_text_string(value):
data[key] = value[:-1] if value[-1] == '/' else value
except Exception as error:
logger.error(str(error))
return data
def get_api_info(self, url, proxy_servers=None, verify=True):
"""Query anaconda api info."""
logger.debug(str((url)))
proxy_servers = proxy_servers or {}
method = self._get_api_info
return self._create_worker(method,
url,
proxy_servers=proxy_servers,
verify=verify)