class WizardAppWidgetStep(traitlets.HasTraits):
"One step of a WizardAppWidget."
class State(Enum):
"""Each step is always in one specific state.
The state is used to determine:
1) how the step is visually presented to the user, and
2) whether the next step is accessible (i.e. reached the SUCCESS state).
App developers are encouraged to use the step states to couple application
logic and interface. In general, all widget changes should trigger
a re-evaluation of the step state, and states also determine whether certain
widgets are enabled or disabled.
A step can be in one of the following states:
INIT: The initial state, usually all widgets disabled.
READY: The step (widget) is ready for user input (some or all widgets enabled).
CONFIGURED: The step is in a consistent configuration awaiting confirmation.
ACTIVE: The step is carrying out a runtime operation.
SUCCESS: A configuration has been confirmed / a runtime operation successfully finished.
FAIL: A runtime operation has failed in an unrecoverable way.
Not all steps must implement all states, for example:
- the first step does not need an INIT state
- a step without runtime process should not have an ACTIVE or FAIL state
- a "review & confirm" step does not require a READY state.
- a step without configuration options (e.g. pure "review & confirm" step)
Important: The next step is only accessible if the current step is within the SUCCESS
state!
"""
INIT = 0 # the step is initialized and all widgets are typically disabled
# The step is correctly configured and can in principle be confirmed.
CONFIGURED = 1 # configuration is valid
READY = 2 # step is ready for user input
ACTIVE = 3 # step is carrying out a runtime operation
SUCCESS = 4 # step has successfully completed
# All error states have negative codes
FAIL = -1 # the step has unrecoverably failed
state = traitlets.UseEnum(State)
auto_advance = traitlets.Bool()
def can_reset(self):
return hasattr(self, "reset")
class Heartbeat(Configurable): #用于节点故障检测
class Status(enum.Enum):
dead = 0
alive = 1
status = traitlets.UseEnum(Status, default_value=Status.dead)
running = traitlets.Bool(default_value=False)
# config
period = traitlets.Float(default_value=0.5).tag(config=True)
def __init__(self, *args, **kwargs):
super(Heartbeat, self).__init__(*args,
**kwargs) # initializes traitlets
self.pulseout = widgets.FloatText(value=time.time()) #发出脉冲
self.pulsein = widgets.FloatText(value=time.time()) #读取引脚脉冲
self.link = widgets.jsdlink((self.pulseout, 'value'),
(self.pulsein, 'value'))
self.start()
def _run(self):
while True:
if not self.running:
break
if self.pulseout.value - self.pulsein.value >= self.period:
self.status = Heartbeat.Status.dead
else:
self.status = Heartbeat.Status.alive
self.pulseout.value = time.time()
time.sleep(self.period)
def start(self):
if self.running:
return
self.running = True
self.thread = threading.Thread(target=self._run)
self.thread.start()
def stop(self):
self.running = False
class AiidaLabApp(traitlets.HasTraits):
"""Manage installation status of an AiiDAlab app.
Arguments:
name (str):
Name of the Aiida lab app.
app_data (dict):
Dictionary containing the app metadata.
aiidalab_apps_path (str):
Path to directory at which the app is expected to be installed.
watch (bool):
If true (default), automatically watch the repository for changes.
"""
path = traitlets.Unicode(allow_none=True, readonly=True)
install_info = traitlets.Unicode()
available_versions = traitlets.List(traitlets.Unicode)
installed_version = traitlets.Union(
[traitlets.Unicode(),
traitlets.UseEnum(AppVersion)])
updates_available = traitlets.Bool(readonly=True, allow_none=True)
busy = traitlets.Bool(readonly=True)
detached = traitlets.Bool(readonly=True, allow_none=True)
compatible = traitlets.Bool(readonly=True, allow_none=True)
@dataclass
class AppRegistryData:
"""Dataclass that contains the app data from the app registry."""
git_url: str
meta_url: str
categories: List[str]
groups: List[str] # appears to be a duplicate of categories?
metainfo: Dict[str, str] = field(default_factory=dict)
gitinfo: Dict[str, str] = field(default_factory=dict)
hosted_on: str = None
class _GitReleaseLine:
"""Utility class to operate on the release line of the app.
A release line is specified via the app url as part of the fragment (after '#').
A release line can be specified either as
a) a commit denoted by a hexadecimal number with either 20 or 40 digits, or
b) a short reference, which can be either a branch or a tag name.
A full ref is the ref as defined in the Git glossary, e.g., 'refs/heads/main'.
A revision is either a full ref or a commit.
"""
def __init__(self, app, line):
self.app = app
self.line = line
match = re.fullmatch(
r'(?P<commit>([0-9a-fA-F]{20}){1,2})|(?P<short_ref>.+)', line)
if not match:
raise ValueError(f"Illegal release line: {line}")
self.commit = match.groupdict()['commit']
self.short_ref = match.groupdict()['short_ref']
assert self.commit or self.short_ref
@property
def _repo(self):
return Repo(self.app.path)
def _resolve_short_ref(self, short_ref):
"""Attempt to resolve the short-ref to a full ref.
For example, 'branch' would be resolved to 'refs/heads/branch'
if 'branch' is a local branch or 'refs/tags/branch' if it was
a tag.
This function returns None if the short-ref cannot be resolved
to a full reference.
"""
# Check if short-ref is among the remote refs:
for ref in self._repo.refs.allkeys():
if re.match(r'refs\/remotes\/(.*)?\/' + short_ref,
ref.decode()):
return ref
# Check if short-ref is a head (branch):
if f'refs/heads/{short_ref}'.encode() in self._repo.refs.allkeys():
return f'refs/heads/{short_ref}'.encode()
# Check if short-ref is a tag:
if f'refs/tags/{short_ref}'.encode() in self._repo.refs.allkeys():
return f'refs/tags/{short_ref}'.encode()
return None
@staticmethod
def _get_sha(obj):
"""Determine the SHA for a given commit object."""
assert isinstance(obj, (Tag, Commit))
return obj.object[1] if isinstance(obj, Tag) else obj.id
def find_versions(self):
"""Find versions available for this release line.
When encountering an ambiguous release line name, i.e.,
a shared branch and tag name, we give preference to the
branch, because that is what git does in this situation.
"""
assert self.short_ref or self.commit
if self.commit: # The release line is a commit.
assert self.commit.encode() in self._repo.object_store
yield self.commit.encode()
else:
ref = self._resolve_short_ref(self.short_ref)
if ref is None:
raise ValueError(
f"Unable to resolve {self.short_ref!r}. "
"Are you sure this is a valid git branch or tag?")
# The release line is a head (branch).
if ref.startswith(b'refs/remotes/'):
ref_commit = self._repo.get_peeled(ref)
all_tags = {
ref
for ref in self._repo.get_refs()
if ref.startswith(b'refs/tags')
}
# Create lookup table from commit -> tags
tags_lookup = defaultdict(set)
for tag in all_tags:
tags_lookup[self._get_sha(self._repo[tag])].add(tag)
# Determine all the tagged commits on the branch (HEAD)
commits_on_head = self._repo.get_walker(
self._repo.refs[ref])
tagged_commits_on_head = [
c.commit.id for c in commits_on_head
if c.commit.id in tags_lookup
]
# Always yield the tip of the branch (HEAD), i.e., the latest commit on the branch.
yield from tags_lookup.get(ref_commit, (ref, ))
# Yield all other tagged commits on the branch:
for commit in tagged_commits_on_head:
if commit != ref_commit:
yield from tags_lookup[commit]
# The release line is a tag.
elif ref.startswith(b'refs/tags/'):
yield ref
def _resolve_commit(self, rev):
"""Map a revision to a commit."""
if len(rev) in (20, 40) and rev in self._repo.object_store:
return rev
return self._get_sha(self._repo[rev])
def resolve_revision(self, commit):
"""Map a given commit to a named version (branch/tag) if possible."""
lookup = defaultdict(set)
for version in self.find_versions():
lookup[self._resolve_commit(version)].add(version)
return lookup.get(commit, {commit})
def _on_release_line(self, rev):
"""Determine whether the release line contains the provided version."""
return rev in [
self._resolve_commit(version)
for version in self.find_versions()
]
def current_revision(self):
"""Return the version currently installed on the release line.
Returns None if the current revision is not on this release line.
"""
current_commit = self._repo.head()
on_release_line = self._on_release_line(current_commit)
if on_release_line:
return list(sorted(self.resolve_revision(current_commit)))[0]
return None # current revision not on the release line
def is_branch(self):
"""Return True if release line is a branch."""
return f'refs/remotes/origin/{self.line}'.encode(
) in self._repo.refs
def __init__(self, name, app_data, aiidalab_apps_path, watch=True):
super().__init__()
if app_data is None:
self._registry_data = None
self._release_line = None
else:
self._registry_data = self.AppRegistryData(**app_data)
parsed_url = urlsplit(self._registry_data.git_url)
self._release_line = self._GitReleaseLine(
self, parsed_url.fragment or AIIDALAB_DEFAULT_GIT_BRANCH)
self.name = name
self.path = os.path.join(aiidalab_apps_path, self.name)
self.refresh_async()
if watch:
self._watch = AiidaLabAppWatch(self)
self._watch.start()
else:
self._watch = None
def __repr__(self):
app_data_argument = None if self._registry_data is None else asdict(
self._registry_data)
return (
f"AiidaLabApp(name={self.name!r}, app_data={app_data_argument!r}, "
f"aiidalab_apps_path={os.path.dirname(self.path)!r})")
@traitlets.default('detached')
def _default_detached(self):
"""Provide default value for detached traitlet."""
if self.is_installed():
modified = self._repo.dirty()
if self._release_line is not None:
revision = self._release_line.current_revision()
return revision is not None and not modified
return True
return None
@traitlets.default('busy')
def _default_busy(self): # pylint: disable=no-self-use
return False
@contextmanager
def _show_busy(self):
"""Apply this decorator to indicate that the app is busy during execution."""
self.set_trait('busy', True)
try:
yield
finally:
self.set_trait('busy', False)
def in_category(self, category):
# One should test what happens if the category won't be defined.
return category in self._registry_data.categories
def is_installed(self):
"""The app is installed if the corresponding folder is present."""
return os.path.isdir(self.path)
def _has_git_repo(self):
"""Check if the app has a .git folder in it."""
try:
Repo(self.path)
return True
except NotGitRepository:
return False
def _install_app_version(self, version):
"""Install a specific app version."""
assert self._registry_data is not None
assert self._release_line is not None
with self._show_busy():
if not re.fullmatch(
r'git:((?P<commit>([0-9a-fA-F]{20}){1,2})|(?P<short_ref>.+))',
version):
raise ValueError(f"Unknown version format: '{version}'")
if not os.path.isdir(self.path): # clone first
url = urldefrag(self._registry_data.git_url).url
check_output(['git', 'clone', url, self.path],
cwd=os.path.dirname(self.path),
stderr=STDOUT)
# Switch to desired version
rev = self._release_line.resolve_revision(
re.sub('git:', '', version)).pop().encode()
if self._release_line.is_branch():
branch = self._release_line.line
check_output(['git', 'checkout', '--force', branch],
cwd=self.path,
stderr=STDOUT)
check_output(['git', 'reset', '--hard', rev],
cwd=self.path,
stderr=STDOUT)
else:
check_output(['git', 'checkout', '--force', rev],
cwd=self.path,
stderr=STDOUT)
self.refresh()
return 'git:' + rev.decode()
def install_app(self, version=None):
"""Installing the app."""
if version is None: # initial installation
version = self._install_app_version(
f'git:{self._release_line.line}')
# switch to compatible version if possible
available_versions = list(self._available_versions())
if available_versions:
return self._install_app_version(version=available_versions[0])
return version
# app already installed, just switch version
return self._install_app_version(version=version)
def update_app(self, _=None):
"""Perform app update."""
assert self._registry_data is not None
try:
if self._remote_update_available():
self._fetch_from_remote()
except AppRemoteUpdateError:
pass
available_versions = list(self._available_versions())
return self.install_app(version=available_versions[0])
def uninstall_app(self, _=None):
"""Perfrom app uninstall."""
# Perform uninstall process.
with self._show_busy():
try:
shutil.rmtree(self.path)
except FileNotFoundError:
raise RuntimeError("App was already uninstalled!")
self.refresh()
def _remote_update_available(self):
"""Check whether there are more commits at the origin (based on the registry)."""
error_message_prefix = "Unable to determine whether remote update is available: "
try: # Obtain reference to git repository.
repo = self._repo
except NotGitRepository as error:
raise AppRemoteUpdateError(f"{error_message_prefix}{error}")
try: # Determine sha of remote-tracking branch from registry.
branch = self._release_line.line
branch_ref = 'refs/heads/' + branch
local_remote_ref = 'refs/remotes/origin/' + branch
remote_sha = self._registry_data.gitinfo[branch_ref]
except AttributeError:
raise AppRemoteUpdateError(
f"{error_message_prefix}app is not registered")
except KeyError:
raise AppRemoteUpdateError(
f"{error_message_prefix}no data about this release line in registry"
)
try: # Determine sha of remote-tracking branch from repository.
local_remote_sha = repo.refs[local_remote_ref.encode()].decode()
except KeyError:
return False # remote ref not found, release line likely not a branch
return remote_sha != local_remote_sha
def _fetch_from_remote(self):
with self._show_busy():
fetch(repo=self._repo,
remote_location=urldefrag(self._registry_data.git_url).url)
def check_for_updates(self):
"""Check whether there is an update available for the installed release line."""
try:
assert not self.detached
remote_update_available = self._remote_update_available()
except (AssertionError, AppRemoteUpdateError):
self.set_trait('updates_available', None)
else:
available_versions = list(self._available_versions())
if len(available_versions) > 0:
local_update_available = self.installed_version != available_versions[
0]
else:
local_update_available = None
self.set_trait('updates_available', remote_update_available
or local_update_available)
def _available_versions(self):
"""Return all available and compatible versions."""
if self.is_installed() and self._release_line is not None:
versions = [
'git:' + ref.decode()
for ref in self._release_line.find_versions()
]
elif self._registry_data is not None:
def is_tag(ref):
return ref.startswith('refs/tags') and '^{}' not in ref
def sort_key(ref):
version = parse(ref[len('refs/tags/'):])
return (not is_tag(ref), version, ref)
versions = [
'git:' + ref for ref in reversed(
sorted(self._registry_data.gitinfo, key=sort_key)) if
is_tag(ref) or ref == f'refs/heads/{self._release_line.line}'
]
else:
versions = []
for version in versions:
if self._is_compatible(version):
yield version
def _installed_version(self):
"""Determine the currently installed version."""
if self.is_installed():
if self._has_git_repo():
modified = self._repo.dirty()
if not (self._release_line is None or modified):
revision = self._release_line.current_revision()
if revision is not None:
return f'git:{revision.decode()}'
return AppVersion.UNKNOWN
return AppVersion.NOT_INSTALLED
@traitlets.default('compatible')
def _default_compatible(self): # pylint: disable=no-self-use
return None
def _is_compatible(self, app_version=None):
"""Determine whether the currently installed version is compatible."""
if app_version is None:
app_version = self.installed_version
def get_version_identifier(version):
"Get version identifier from version (e.g. git:refs/tags/v1.0.0 -> v1.0.0)."
if version.startswith('git:refs/tags/'):
return version[len('git:refs/tags/'):]
if version.startswith('git:refs/heads/'):
return version[len('git:refs/heads/'):]
if version.startswith('git:refs/remotes/'): # remote branch
return re.sub(r'git:refs\/remotes\/(.+?)\/', '', version)
return version
class RegexMatchSpecifierSet:
"""Interpret 'invalid' specifier sets as regular expression pattern."""
def __init__(self, specifiers=''):
self.specifiers = specifiers
def __contains__(self, version):
return re.match(self.specifiers, version) is not None
def specifier_set(specifiers=''):
try:
return SpecifierSet(specifiers=specifiers, prereleases=True)
except InvalidSpecifier:
return RegexMatchSpecifierSet(specifiers=specifiers)
def fulfilled(requirements, packages):
for requirement in requirements:
if not any(
package.fulfills(requirement) for package in packages):
logging.debug(
f"{self.name}({app_version}): missing requirement '{requirement}'"
) # pylint: disable=logging-fstring-interpolation
return False
return True
# Retrieve and convert the compatibility map from the app metadata.
try:
compat_map = self.metadata.get('requires', {'': []})
compat_map = {
specifier_set(app_version): [Requirement(r) for r in reqs]
for app_version, reqs in compat_map.items()
}
except RuntimeError: # not registered
return None # unable to determine compatibility
else:
if isinstance(app_version, str):
app_version_identifier = get_version_identifier(app_version)
matching_specs = [
app_spec for app_spec in compat_map
if app_version_identifier in app_spec
]
packages = find_installed_packages()
return any(
fulfilled(compat_map[spec], packages)
for spec in matching_specs)
return None # compatibility indetermined since the app is not installed
@throttled(calls_per_second=1)
def refresh(self):
"""Refresh app state."""
with self._show_busy():
with self.hold_trait_notifications():
self.available_versions = list(self._available_versions())
self.installed_version = self._installed_version()
self.set_trait('compatible', self._is_compatible())
if self.is_installed() and self._has_git_repo():
self.installed_version = self._installed_version()
modified = self._repo.dirty()
self.set_trait(
'detached',
self.installed_version is AppVersion.UNKNOWN
or modified)
self.check_for_updates()
else:
self.set_trait('updates_available', None)
self.set_trait('detached', None)
def refresh_async(self):
"""Asynchronized (non-blocking) refresh of the app state."""
refresh_thread = Thread(target=self.refresh)
refresh_thread.start()
@property
def metadata(self):
"""Return metadata dictionary. Give the priority to the local copy (better for the developers)."""
if self._registry_data is not None and self._registry_data.metainfo:
return dict(self._registry_data.metainfo)
if self.is_installed():
try:
with open(os.path.join(self.path,
'metadata.json')) as json_file:
return json.load(json_file)
except IOError:
return dict()
raise RuntimeError(
f"Requested app '{self.name}' is not installed and is also not registered on the app registry."
)
def _get_from_metadata(self, what):
"""Get information from metadata."""
try:
return "{}".format(self.metadata[what])
except KeyError:
if not os.path.isfile(os.path.join(self.path, 'metadata.json')):
return '({}) metadata.json file is not present'.format(what)
return 'the field "{}" is not present in metadata.json file'.format(
what)
@property
def authors(self):
return self._get_from_metadata('authors')
@property
def description(self):
return self._get_from_metadata('description')
@property
def title(self):
return self._get_from_metadata('title')
@property
def url(self):
"""Provide explicit link to Git repository."""
return getattr(self._registry_data, "git_url", None)
@property
def more(self):
return """<a href=./single_app.ipynb?app={}>Manage App</a>""".format(
self.name)
@property
def _repo(self):
"""Returns Git repository."""
if not self.is_installed():
raise AppNotInstalledException("The app is not installed")
return Repo(self.path)
class Axis(_HasState):
class Status(enum.Enum):
"""
State transitions
NO_LIMITS -> STAGED_CALCULATING_LIMITS -> CALCULATING_LIMITS -> CALCULATED_LIMITS -> READY
when expression changes:
STAGED_CALCULATING_LIMITS:
calculation.cancel()
->NO_LIMITS
CALCULATING_LIMITS:
calculation.cancel()
->NO_LIMITS
when min/max changes:
STAGED_CALCULATING_LIMITS:
calculation.cancel()
->NO_LIMITS
CALCULATING_LIMITS:
calculation.cancel()
->NO_LIMITS
"""
NO_LIMITS = 1
STAGED_CALCULATING_LIMITS = 2
CALCULATING_LIMITS = 3
CALCULATED_LIMITS = 4
READY = 5
EXCEPTION = 6
ABORTED = 7
status = traitlets.UseEnum(Status, Status.NO_LIMITS)
df = traitlets.Instance(vaex.dataframe.DataFrame)
expression = Expression()
slice = traitlets.CInt(None, allow_none=True)
min = traitlets.CFloat(None, allow_none=True)
max = traitlets.CFloat(None, allow_none=True)
bin_centers = traitlets.Any()
shape = traitlets.CInt(None, allow_none=True)
shape_default = traitlets.CInt(64)
_calculation = traitlets.Any(None, allow_none=True)
exception = traitlets.Any(None, allow_none=True)
_status_change_delay = traitlets.Float(0)
def __init__(self, **kwargs):
super().__init__(**kwargs)
if self.min is not None and self.max is not None:
self.status = Axis.Status.READY
self._calculate_centers()
else:
self.computation()
self.observe(self.on_change_expression, 'expression')
self.observe(self.on_change_shape, 'shape')
self.observe(self.on_change_shape_default, 'shape_default')
def __repr__(self):
def myrepr(value, key):
if isinstance(value, vaex.expression.Expression):
return str(value)
return value
args = ', '.join('{}={}'.format(key, myrepr(getattr(self, key), key))
for key in self.traits().keys()
if key != 'df' and not key.startswith('_'))
return '{}({})'.format(self.__class__.__name__, args)
@property
def has_missing_limit(self):
# return not self.df.is_category(self.expression) and (self.min is None or self.max is None)
return (self.min is None or self.max is None)
def on_change_expression(self, change):
self.min = None
self.max = None
self.status = Axis.Status.NO_LIMITS
if self._calculation is not None:
self._cancel_computation()
self.computation()
def on_change_shape(self, change):
if self.min is not None and self.max is not None:
self._calculate_centers()
def on_change_shape_default(self, change):
if self.min is not None and self.max is not None:
self._calculate_centers()
def _cancel_computation(self):
self._continue_calculation = False
@traitlets.observe('min', 'max')
def on_change_limits(self, change):
if self.min is not None and self.max is not None:
self._calculate_centers()
if self.status == Axis.Status.NO_LIMITS:
if self.min is not None and self.max is not None:
self.status = Axis.Status.READY
elif self.status == Axis.Status.READY:
if self.min is None or self.max is None:
self.status = Axis.Status.NO_LIMITS
else:
# in this case, grids may want to be computed
# this happens when a user change min/max
pass
else:
if self._calculation is not None:
self._cancel_computation()
if self.min is not None and self.max is not None:
self.status = Axis.Status.READY
else:
self.status = Axis.Status.NO_LIMITS
else:
# in this case we've set min/max after the calculation
assert self.min is not None or self.max is not None
@vaex.jupyter.debounced(delay_seconds=0.1,
reentrant=False,
on_error=_HasState._error)
async def computation(self):
categorical = self.df.is_category(self.expression)
if categorical:
N = self.df.category_count(self.expression)
self.min, self.max = -0.5, N - 0.5
# centers = np.arange(N)
# self.shape = N
self._calculate_centers()
self.status = Axis.Status.READY
else:
try:
self._continue_calculation = True
self._calculation = self.df.minmax(self.expression,
delay=True,
progress=self._progress)
self.df.widget.execute_debounced()
# keep a nearly reference to this, since awaits (which trigger the execution, AND reset of this future) may change it this
execute_prehook_future = self.df.widget.execute_debounced.pre_hook_future
async with self._state_change_to(
Axis.Status.STAGED_CALCULATING_LIMITS):
pass
async with self._state_change_to(
Axis.Status.CALCULATING_LIMITS):
await execute_prehook_future
async with self._state_change_to(
Axis.Status.CALCULATED_LIMITS):
vmin, vmax = await self._calculation
# indicate we are done with the calculation
self._calculation = None
if not self._continue_calculation:
assert self.status == Axis.Status.READY
async with self._state_change_to(Axis.Status.READY):
self.min, self.max = vmin, vmax
self._calculate_centers()
except vaex.execution.UserAbort:
# probably means expression or min/max changed, we don't have to take action
pass
except asyncio.CancelledError:
pass
def _progress(self, f):
# we use the progres callback to cancel as calculation
return self._continue_calculation
def _calculate_centers(self):
categorical = self.df.is_category(self.expression)
if categorical:
N = self.df.category_count(self.expression)
centers = np.arange(N)
self.shape = N
else:
centers = self.df.bin_centers(self.expression,
[self.min, self.max],
shape=self.shape
or self.shape_default)
self.bin_centers = centers
class GridCalculator(_HasState):
'''A grid is responsible for scheduling the grid calculations and possible slicing'''
class Status(enum.Enum):
VOID = 1
STAGED_CALCULATION = 3
CALCULATING = 4
READY = 9
status = traitlets.UseEnum(Status, Status.VOID)
df = traitlets.Instance(vaex.dataframe.DataFrame)
models = traitlets.List(traitlets.Instance(DataArray))
_calculation = traitlets.Any(None, allow_none=True)
_debug = traitlets.Bool(False)
def __init__(self, df, models):
super().__init__(df=df, models=[])
self._callbacks_regrid = []
self._callbacks_slice = []
for model in models:
self.model_add(model)
self._testing_exeception_regrid = False # used for testing, to throw an exception
self._testing_exeception_reslice = False # used for testing, to throw an exception
# def model_remove(self, model, regrid=True):
# index = self.models.index(model)
# del self.models[index]
# del self._callbacks_regrid[index]
# del self._callbacks_slice[index]
def model_add(self, model):
self.models = self.models + [model]
if model.status == DataArray.Status.NEEDS_CALCULATING_GRID:
if self._calculation is not None:
self._cancel_computation()
self.computation()
def on_status_changed(change):
if change.owner.status == DataArray.Status.NEEDS_CALCULATING_GRID:
if self._calculation is not None:
self._cancel_computation()
self.computation()
model.observe(on_status_changed, 'status')
# TODO: if we listen to the same axis twice it will trigger twice
for axis in model.axes:
axis.observe(lambda change: self.reslice(), 'slice')
# self._callbacks_regrid.append(model.signal_regrid.connect(self.on_regrid))
# self._callbacks_slice.append(model.signal_slice.connect(self.reslice))
assert model.df == self.df
# @vaex.jupyter.debounced(delay_seconds=0.05, reentrant=False)
# def reslice_debounced(self):
# self.reslice()
def reslice(self, source_model=None):
if self._testing_exeception_reslice:
raise RuntimeError("test:reslice")
coords = []
selection_was_list, [selections
] = vaex.utils.listify(self.models[0].selection)
selections = [
k for k in selections if k is None or self.df.has_selection(k)
]
for model in self.models:
subgrid = self.grid
if not selection_was_list:
subgrid = subgrid[0]
subgrid_sliced = self.grid
if not selection_was_list:
subgrid_sliced = subgrid_sliced[0]
axis_index = 1 if selection_was_list else 0
has_slice = False
dims = ["selection"] if selection_was_list else []
coords = [selections.copy()] if selection_was_list else []
mins = []
maxs = []
for other_model in self.models:
if other_model == model: # simply skip these axes
# for expression, shape, limit, slice_index in other_model.bin_parameters():
for axis in other_model.axes:
axis_index += 1
dims.append(str(axis.expression))
coords.append(axis.bin_centers)
mins.append(axis.min)
maxs.append(axis.max)
else:
# for expression, shape, limit, slice_index in other_model.bin_parameters():
for axis in other_model.axes:
if axis.slice is not None:
subgrid_sliced = subgrid_sliced.__getitem__(
tuple([slice(None)] * axis_index +
[axis.slice])).copy()
subgrid = np.sum(subgrid, axis=axis_index)
has_slice = True
else:
subgrid_sliced = np.sum(subgrid_sliced,
axis=axis_index)
subgrid = np.sum(subgrid, axis=axis_index)
grid = xarray.DataArray(subgrid, dims=dims, coords=coords)
# +1 to skip the selection axis
dim_offset = 1 if selection_was_list else 0
for i, (vmin, vmax) in enumerate(zip(mins, maxs)):
grid.coords[dims[i + dim_offset]].attrs['min'] = vmin
grid.coords[dims[i + dim_offset]].attrs['max'] = vmax
model.grid = grid
if has_slice:
model.grid_sliced = xarray.DataArray(subgrid_sliced)
else:
model.grid_sliced = None
def _regrid_error(self, e):
try:
self._error(e)
for model in self.models:
model._error(e)
for model in self.models:
model.exception = e
model.status = vaex.jupyter.model.DataArray.Status.EXCEPTION
except Exception as e2:
print(e2)
def on_regrid(self, ignore=None):
self.regrid()
@vaex.jupyter.debounced(delay_seconds=0.5,
reentrant=False,
on_error=_regrid_error)
async def computation(self):
try:
logger.debug('Starting grid computation')
# vaex.utils.print_stack_trace()
if self._testing_exeception_regrid:
raise RuntimeError("test:regrid")
if not self.models:
return
binby = []
shapes = []
limits = []
selection = self.models[0].selection
selection_was_list, [selections] = vaex.utils.listify(
self.models[0].selection)
selections = [
k for k in selections if k is None or self.df.has_selection(k)
]
for model in self.models:
if model.selection != selection:
raise ValueError(
'Selections for all models should be the same')
for axis in model.axes:
binby.append(axis.expression)
limits.append([axis.min, axis.max])
shapes.append(axis.shape or axis.shape_default)
selections = [
k for k in selections if k is None or self.df.has_selection(k)
]
self._continue_calculation = True
logger.debug('Setting up grid computation...')
self._calculation = self.df.count(binby=binby,
shape=shapes,
limits=limits,
selection=selections,
progress=self.progress,
delay=True)
logger.debug('Setting up grid computation done tasks=%r',
self.df.executor.tasks)
logger.debug('Schedule debounced execute')
self.df.widget.execute_debounced()
# keep a nearly reference to this, since awaits (which trigger the execution, AND reset of this future) may change it this
execute_prehook_future = self.df.widget.execute_debounced.pre_hook_future
async with contextlib.AsyncExitStack() as stack:
for model in self.models:
await stack.enter_async_context(
model._state_change_to(
DataArray.Status.STAGED_CALCULATING_GRID))
async with contextlib.AsyncExitStack() as stack:
for model in self.models:
await stack.enter_async_context(
model._state_change_to(
DataArray.Status.CALCULATING_GRID))
await execute_prehook_future
async with contextlib.AsyncExitStack() as stack:
for model in self.models:
await stack.enter_async_context(
model._state_change_to(
DataArray.Status.CALCULATED_GRID))
# first assign to local
grid = await self._calculation
# indicate we are done with the calculation
self._calculation = None
# raise asyncio.CancelledError("User abort")
async with contextlib.AsyncExitStack() as stack:
for model in self.models:
await stack.enter_async_context(
model._state_change_to(DataArray.Status.READY))
self.grid = grid
self.reslice()
except vaex.execution.UserAbort:
pass # a user changed the limits or expressions
except asyncio.CancelledError:
pass # cancelled...
def _cancel_computation(self):
logger.debug('Cancelling grid computation')
self._continue_calculation = False
def progress(self, f):
return self._continue_calculation and all(
[model.on_progress_grid(f) for model in self.models])
class DataArray(_HasState):
class Status(enum.Enum):
MISSING_LIMITS = 1
STAGED_CALCULATING_LIMITS = 3
CALCULATING_LIMITS = 4
CALCULATED_LIMITS = 5
NEEDS_CALCULATING_GRID = 6
STAGED_CALCULATING_GRID = 7
CALCULATING_GRID = 8
CALCULATED_GRID = 9
READY = 10
EXCEPTION = 11
status = traitlets.UseEnum(Status, Status.MISSING_LIMITS)
status_text = traitlets.Unicode('Initializing')
exception = traitlets.Any(None)
df = traitlets.Instance(vaex.dataframe.DataFrame)
axes = traitlets.List(traitlets.Instance(Axis), [])
grid = traitlets.Instance(xarray.DataArray, allow_none=True)
grid_sliced = traitlets.Instance(xarray.DataArray, allow_none=True)
shape = traitlets.CInt(64)
selection = traitlets.Any(None)
def __init__(self, **kwargs):
super(DataArray, self).__init__(**kwargs)
self.signal_slice = vaex.events.Signal()
self.signal_regrid = vaex.events.Signal()
self.signal_grid_progress = vaex.events.Signal()
self.observe(lambda change: self.signal_regrid.emit(), 'selection')
self._on_axis_status_change()
# keep a set of axis that need new limits
self._dirty_axes = set()
for axis in self.axes:
assert axis.df is self.df, "axes should have the same dataframe"
traitlets.link((self, 'shape'), (axis, 'shape_default'))
axis.observe(self._on_axis_status_change, 'status')
axis.observe(lambda _: self.signal_slice.emit(self), ['slice'])
def on_change_min_max(change):
if change.owner.status == Axis.Status.READY:
# this indicates a user changed the min/max
self.status = DataArray.Status.NEEDS_CALCULATING_GRID
axis.observe(on_change_min_max, ['min', 'max'])
self._on_axis_status_change()
self.df.signal_selection_changed.connect(self._on_change_selection)
def _on_change_selection(self, df, name):
# TODO: check if the selection applies to us
self.status = DataArray.Status.NEEDS_CALCULATING_GRID
async def _allow_state_change_cancel(self):
self._allow_state_change.release()
def _on_axis_status_change(self, change=None):
missing_limits = [
axis for axis in self.axes if axis.status == Axis.Status.NO_LIMITS
]
staged_calculating_limits = [
axis for axis in self.axes
if axis.status == Axis.Status.STAGED_CALCULATING_LIMITS
]
calculating_limits = [
axis for axis in self.axes
if axis.status == Axis.Status.CALCULATING_LIMITS
]
calculated_limits = [
axis for axis in self.axes
if axis.status == Axis.Status.CALCULATED_LIMITS
]
def names(axes):
return ", ".join([str(axis.expression) for axis in axes])
if staged_calculating_limits:
self.status = DataArray.Status.STAGED_CALCULATING_LIMITS
self.status_text = 'Staged limit computation for {}'.format(
names(staged_calculating_limits))
elif missing_limits:
self.status = DataArray.Status.MISSING_LIMITS
self.status_text = 'Missing limits for {}'.format(
names(missing_limits))
elif calculating_limits:
self.status = DataArray.Status.CALCULATING_LIMITS
self.status_text = 'Computing limits for {}'.format(
names(calculating_limits))
elif calculated_limits:
self.status = DataArray.Status.CALCULATED_LIMITS
self.status_text = 'Computed limits for {}'.format(
names(calculating_limits))
else:
assert all(
[axis.status == Axis.Status.READY for axis in self.axes])
self.status = DataArray.Status.NEEDS_CALCULATING_GRID
@traitlets.observe('status')
def _on_change_status(self, change):
if self.status == DataArray.Status.EXCEPTION:
self.status_text = f'Exception: {self.exception}'
elif self.status == DataArray.Status.NEEDS_CALCULATING_GRID:
self.status_text = 'Grid needs to be calculated'
elif self.status == DataArray.Status.STAGED_CALCULATING_GRID:
self.status_text = 'Staged grid computation'
elif self.status == DataArray.Status.CALCULATING_GRID:
self.status_text = 'Calculating grid'
elif self.status == DataArray.Status.CALCULATED_GRID:
self.status_text = 'Calculated grid'
elif self.status == DataArray.Status.READY:
self.status_text = 'Ready'
# GridCalculator can change the status
# self._update_grid()
# self.status_text = 'Computing limits for {}'.format(names(missing_limits))
@property
def has_missing_limits(self):
return any([axis.has_missing_limit for axis in self.axes])
def on_progress_grid(self, f):
return all(self.signal_grid_progress.emit(f))
class AiidaLabApp(traitlets.HasTraits):
"""Manage installation status of an AiiDAlab app.
Arguments:
name (str):
Name of the Aiida lab app.
app_data (dict):
Dictionary containing the app metadata.
aiidalab_apps_path (str):
Path to directory at which the app is expected to be installed.
watch (bool):
If true (default), automatically watch the repository for changes.
"""
path = traitlets.Unicode(allow_none=True, readonly=True)
install_info = traitlets.Unicode()
available_versions = traitlets.List(traitlets.Unicode)
installed_version = traitlets.Union(
[traitlets.Unicode(), traitlets.UseEnum(AppVersion)]
)
remote_update_status = traitlets.UseEnum(
AppRemoteUpdateStatus, readonly=True, allow_none=True
)
has_prereleases = traitlets.Bool()
include_prereleases = traitlets.Bool()
busy = traitlets.Bool(readonly=True)
detached = traitlets.Bool(readonly=True, allow_none=True)
compatible = traitlets.Bool(readonly=True, allow_none=True)
compatibility_info = traitlets.Dict()
def __init__(self, name, app_data, aiidalab_apps_path, watch=True):
self._app = _AiidaLabApp.from_id(
name, registry_entry=app_data, apps_path=aiidalab_apps_path
)
super().__init__()
self.name = self._app.name
self.logo = self._app.metadata["logo"]
self.categories = self._app.metadata["categories"]
self.is_installed = self._app.is_installed
self.path = str(self._app.path)
self.refresh_async()
if watch:
self._watch = AiidaLabAppWatch(self)
self._watch.start()
else:
self._watch = None
def __str__(self):
return f"<AiidaLabApp name='{self._app.name}'>"
@traitlets.default("include_prereleases")
def _default_include_prereleases(self):
"Provide default value for include_prereleases trait." ""
return False
@traitlets.observe("include_prereleases")
def _observe_include_prereleases(self, change):
if change["old"] != change["new"]:
self.refresh()
@traitlets.default("detached")
def _default_detached(self):
"""Provide default value for detached traitlet."""
if self.is_installed():
return self._app.dirty() or self._installed_version() is AppVersion.UNKNOWN
return None
@traitlets.default("busy")
def _default_busy(self): # pylint: disable=no-self-use
return False
@contextmanager
def _show_busy(self):
"""Apply this decorator to indicate that the app is busy during execution."""
self.set_trait("busy", True)
try:
yield
finally:
self.set_trait("busy", False)
def in_category(self, category):
# One should test what happens if the category won't be defined.
return category in self.categories
def _has_git_repo(self):
"""Check if the app has a .git folder in it."""
try:
Repo(self.path)
return True
except NotGitRepository:
return False
def install_app(self, version=None, stdout=None):
"""Installing the app."""
with self._show_busy():
self._app.install(
version=version, stdout=stdout, prereleases=self.include_prereleases
)
FIND_INSTALLED_PACKAGES_CACHE.clear()
self.refresh()
return self._installed_version()
def update_app(self, _=None, stdout=None):
"""Perform app update."""
with self._show_busy():
# Installing with version=None automatically selects latest
# available version.
version = self.install_app(version=None, stdout=stdout)
FIND_INSTALLED_PACKAGES_CACHE.clear()
self.refresh()
return version
def uninstall_app(self, _=None):
"""Perfrom app uninstall."""
# Perform uninstall process.
with self._show_busy():
self._app.uninstall()
self.refresh()
def _installed_version(self):
"""Determine the currently installed version."""
return self._app.installed_version()
@traitlets.default("compatible")
def _default_compatible(self): # pylint: disable=no-self-use
return None
def _is_compatible(self, app_version):
"""Determine whether the specified version is compatible."""
try:
incompatibilities = dict(
self._app.find_incompatibilities(version=app_version)
)
self.compatibility_info = {
app_version: [
f"({eco_system}) {requirement}"
for eco_system, requirement in incompatibilities.items()
]
}
return not any(incompatibilities)
except KeyError:
return None # compatibility indetermined for given version
def _remote_update_status(self):
"""Determine whether there are updates available.
For this the app must be installed in a known version and there must be
available (and compatible) versions.
"""
installed_version = self._installed_version()
if installed_version not in (AppVersion.UNKNOWN, AppVersion.NOT_INSTALLED):
available_versions = list(self.available_versions)
if len(available_versions):
return self._installed_version() != available_versions[0]
return False
def _refresh_versions(self):
self.installed_version = self._app.installed_version()
self.include_prereleases = self.include_prereleases or (
isinstance(self.installed_version, str)
and parse(self.installed_version).is_prerelease
)
all_available_versions = list(self._app.available_versions())
self.has_prereleases = any(
parse(version).is_prerelease for version in all_available_versions
)
if self._app.is_registered:
self.available_versions = [
version
for version in all_available_versions
if self.include_prereleases or not parse(version).is_prerelease
]
@throttled(calls_per_second=1)
def refresh(self):
"""Refresh app state."""
with self._show_busy():
with self.hold_trait_notifications():
self._refresh_versions()
self.set_trait(
"compatible", self._is_compatible(self.installed_version)
)
self.set_trait(
"remote_update_status",
self._app.remote_update_status(
prereleases=self.include_prereleases
),
)
self.set_trait(
"detached",
(self.installed_version is AppVersion.UNKNOWN)
if (self._has_git_repo() and self._app.is_registered)
else None,
)
def refresh_async(self):
"""Asynchronized (non-blocking) refresh of the app state."""
refresh_thread = Thread(target=self.refresh)
refresh_thread.start()
@property
def metadata(self):
"""Return metadata dictionary. Give the priority to the local copy (better for the developers)."""
return self._app.metadata
def _get_from_metadata(self, what):
"""Get information from metadata."""
try:
return f"{self._app.metadata[what]}"
except KeyError:
return f'Field "{what}" is not present in app metadata.'
@property
def authors(self):
return self._get_from_metadata("authors")
@property
def description(self):
return self._get_from_metadata("description")
@property
def title(self):
return self._get_from_metadata("title")
@property
def url(self):
"""Provide explicit link to Git repository."""
return self._get_from_metadata("external_url")
@property
def more(self):
return f"""<a href=./single_app.ipynb?app={self.name}>Manage App</a>"""
@property
def _repo(self):
"""Returns Git repository."""
if not self.is_installed():
raise AppNotInstalledException("The app is not installed")
return Repo(self.path)
class WizardAppStep(traitlets.HasTraits):
"One step of a WizardApp."
state = traitlets.UseEnum(WizardApp.State)
auto_next = traitlets.Bool()