def ensure_local(index_url, file_path, local_cache_path,
force=False, progress_advance=None):
localfiles = LocalFiles(local_cache_path,
serverfiles=ServerFiles(server=index_url))
if force:
localfiles.download(*file_path, callback=progress_advance)
return localfiles.localpath_download(*file_path, callback=progress_advance)
def ensure_local(index_url, file_path, local_cache_path,
force=False, progress_advance=None):
localfiles = LocalFiles(local_cache_path,
serverfiles=ServerFiles(server=index_url))
if force:
localfiles.download(*file_path, callback=progress_advance)
return localfiles.localpath_download(*file_path, callback=progress_advance)
class UDPipeModels:
server_url = "https://file.biolab.si/files/udpipe/"
def __init__(self):
dir_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"udpipe_models"
)
self.serverfiles = ServerFiles(self.server_url)
self.localfiles = LocalFiles(dir_path, serverfiles=self.serverfiles)
def __getitem__(self, language):
file_name = self._find_file(language_to_name(language))
return self.localfiles.localpath_download(file_name)
@property
def model_files(self):
try:
return self.serverfiles.listfiles()
except ConnectionError:
return self.localfiles.listfiles()
def _find_file(self, language):
return next(filter(lambda f: file_to_name(f).startswith(language),
map(lambda f: f[0], self.model_files)))
@property
def supported_languages(self):
return list(map(lambda f: file_to_language(f[0]), self.model_files))
def __init__(self):
dir_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"udpipe_models"
)
self.serverfiles = ServerFiles(self.server_url)
self.localfiles = LocalFiles(dir_path, serverfiles=self.serverfiles)
def ensure_local(domain, filename, force=False, progress_advance=None):
localfiles = LocalFiles(local_cache_path(),
serverfiles=ServerFiles(server=INDEX_URL))
if force:
localfiles.download(domain, filename, callback=progress_advance)
return localfiles.localpath_download(
domain, filename, callback=progress_advance)
def SubmitDownloadTask(self, domain, filename):
"""
Submit the (domain, filename) to be downloaded/updated.
"""
self.cancelButton.setEnabled(True)
index = self.updateItemIndex(domain, filename)
_, tree_item, opt_widget = self.updateItems[index]
sf = LocalFiles(serverfiles.PATH, serverfiles.ServerFiles())
task = DownloadTask(domain, filename, sf)
self.progress.adjustRange(0, 100)
pb = ItemProgressBar(self.filesView)
pb.setRange(0, 100)
pb.setTextVisible(False)
task.advanced.connect(pb.advance)
task.advanced.connect(self.progress.advance)
task.finished.connect(pb.hide)
task.finished.connect(self.onDownloadFinished, Qt.QueuedConnection)
task.exception.connect(self.onDownloadError, Qt.QueuedConnection)
self.filesView.setItemWidget(tree_item, 2, pb)
# Clear the text so it does not show behind the progress bar.
tree_item.setData(2, Qt.DisplayRole, "")
pb.show()
# Disable the options widget
opt_widget.setEnabled(False)
self._tasks.append(task)
self.executor.submit(task)
def tag_list():
"""List of available tags and their pretty-print with indices"""
server_url = "http://butler.fri.uni-lj.si/datasets/"
PATH = os.path.join(data_dir(), "datasets")
local_files = LocalFiles(PATH, serverfiles=ServerFiles(server=server_url))
local_info = local_files.serverfiles.allinfo()
nested_tags = [i["tags"] for i in local_info.values() if i["tags"]]
all_tags = sorted(list(set(itertools.chain(*nested_tags))))
w = max(len(t) for t in all_tags)
n = int(75 / (w + 5))
s = [
"{:>3}-{:<{width}}".format(i, t, width=w)
for i, t in enumerate(all_tags)
]
c = "\n".join(["".join(s[x:x + n]) for x in range(0, len(s), n)])
return all_tags, c
from serverfiles import LocalFiles, ServerFiles
from Orange.data import Table, Domain, StringVariable, DiscreteVariable
from Orange.data import filter as table_filter
from Orange.misc.environ import data_dir
from orangecontrib.bioinformatics.widgets.utils.data import TableAnnotation
domain = 'geo'
_local_cache_path = os.path.join(data_dir(), domain)
_all_info_file = os.path.join(_local_cache_path, '__INFO__')
_server_url = 'http://download.biolab.si/datasets/geo/'
pubmed_url = 'http://www.ncbi.nlm.nih.gov/pubmed/{}'
server_files = ServerFiles(server=_server_url)
local_files = LocalFiles(_local_cache_path, serverfiles=server_files)
def is_cached(gds_id):
return os.path.exists(os.path.join(_local_cache_path, gds_id + '.tab'))
def info_cache(f):
"""Store content of __INFO__ file locally."""
def wrapper():
if not os.path.isfile(_all_info_file) or os.path.getsize(_all_info_file) == 0:
with open(_all_info_file, 'w') as fp:
json.dump(list(server_files.allinfo().items()), fp)
return f()
def list_local(self):
# type: () -> Dict[Tuple[str, str], dict]
return LocalFiles(self.local_cache_path).allinfo()
def commit_cached(self, prefix, filename):
path = LocalFiles(self.local_cache_path).localpath(prefix, filename)
self.Outputs.data.send(Orange.data.Table(path))
def commit_cached(self, file_path):
path = LocalFiles(self.local_cache_path).localpath(*file_path)
self.load_and_output(path)
def list_local(path: str) -> Dict[Tuple[str, ...], dict]:
return LocalFiles(path).allinfo()
def commit_cached(self, file_path):
path = LocalFiles(self.local_cache_path).localpath(*file_path)
self.Outputs.data.send(self.load_data(path))
def commit_cached(self, file_path):
path = LocalFiles(self.local_cache_path).localpath(*file_path)
self.Outputs.data.send(Orange.data.Table(path))
def commit_cached(self, prefix, filename):
path = LocalFiles(local_cache_path()).localpath(prefix, filename)
self.send("Data", Orange.data.Table(path))
def SetFilesList(self, serverInfo):
"""
Set the files to show.
"""
self.setEnabled(True)
localInfo = LocalFiles(serverfiles.PATH).allinfo()
all_tags = set()
self.filesView.clear()
self.updateItems = []
for item in join_info_dict(localInfo, serverInfo):
tree_item = UpdateTreeWidgetItem(item)
options_widget = UpdateOptionsWidget(item.state)
options_widget.item = item
options_widget.installClicked.connect(
partial(self.SubmitDownloadTask, item.domain, item.filename))
options_widget.removeClicked.connect(
partial(self.SubmitRemoveTask, item.domain, item.filename))
self.updateItems.append((item, tree_item, options_widget))
all_tags.update(item.tags)
self.filesView.addTopLevelItems(
[tree_item for _, tree_item, _ in self.updateItems])
for item, tree_item, options_widget in self.updateItems:
self.filesView.setItemWidget(tree_item, 0, options_widget)
# Add an update button if the file is updateable
if item.state == OUTDATED:
button = QToolButton(None,
text="Update",
maximumWidth=120,
maximumHeight=30)
if sys.platform == "darwin":
button.setAttribute(Qt.WA_MacSmallSize)
button.clicked.connect(
partial(self.SubmitDownloadTask, item.domain,
item.filename))
self.filesView.setItemWidget(tree_item, 2, button)
self.progress.advance()
self.filesView.setColumnWidth(0, self.filesView.sizeHintForColumn(0))
for column in range(1, 4):
contents_hint = self.filesView.sizeHintForColumn(column)
header_hint = self.filesView.header().sectionSizeHint(column)
width = max(min(contents_hint, 400), header_hint)
self.filesView.setColumnWidth(column, width)
self.lineEditFilter.setItems(
[hint for hint in sorted(all_tags) if not hint.startswith("#")])
self.SearchUpdate()
self.UpdateInfoLabel()
self.toggleButtons()
self.cancelButton.setEnabled(False)
self.progress.setRange(0, 0)
