class WalkoffSignal(object):
_signals = {}
def __init__(self, name, event_type, loggable=True, message=''):
self.name = name
self.signal = Signal(name)
self.event_type = event_type
if loggable:
signal_callback = partial(add_entry_to_case,
data='',
event_type=event_type.name,
entry_message=message,
message_name=name)
self.connect(signal_callback, weak=False)
def send(self, sender, **kwargs):
self.signal.send(sender, **kwargs)
def connect(self, func, weak=True):
self.signal.connect(func)
if not weak:
WalkoffSignal._store_callback(func)
return func
@classmethod
def _store_callback(cls, func):
"""
Stores callbacks so they aren't garbage collected and the weak references of the signals disappear
"""
cls._signals[id(func)] = func
class EventService(Service):
def __init__(self, env):
"""event service"""
super(EventService, self).__init__(env)
self._channel = Signal('event_channel')
def subscribe(self, func, event_type=None ):
'''
:param func: def func(event_type, **kwarg):
pass
:param event_filter: option
:return:
'''
# sender = event_type or ANY
# weak = True
# if isinstance(event_type, basestring):
# weak = False
# self._channel.connect(func, sender, weak)
sender = event_type or ANY
self._channel.connect(func, sender)
def unsubscribe(self, func):
self._channel.disconnect(func)
def publish(self, event_type, **kwarg):
self._channel.send(event_type, **kwarg)
class CSession:
def __init__(self, name, start, end):
self.name = name
self.uid = uuid.uuid1()
self.start = datetime.strptime(start, DT_FORMAT)
self.end = datetime.strptime(end, DT_FORMAT)
self.on_live = Signal('on-live')
self.on_end = Signal('on-end')
self.liveURL = None
self._teacher = None
self.popups = []
@property
def teacher(self):
return self._teacher
@teacher.setter
def teacher(self, t):
self.on_live.connect(t.live_recv)
self.on_end.connect(t.session_end_recv)
self._teacher = t
def live(self):
self.on_live.send(self, msg="hello")
def liveEnd(self):
self.on_end.send(self, msg="bye")
def openLive(self):
webbrowser.open_new(self.liveURL)
class WalkoffSignal(object):
"""A signal to send Walkoff data
The class is a wrapper around a blinker.Signal
Attributes:
name (str): The name of the signal
signal (Signal): The signal object which sends the event and data
event_type (EventType): The event type of this signal
is_sent_to_interfaces (bool, optional): Should this event get sent to the interface dispatcher? Defaults to True
message (str): Human readable message for this event
Args:
name (str): The name of the signal
event_type (EventType): The event type of this signal
send_to_interfaces (bool, optional): Should this event get sent to the interface dispatcher? Defaults to True
message (str, optional): Human readable message for this event. Defaults to empty string
"""
_signals = {}
def __init__(self, name, event_type, send_to_interfaces=True, message=''):
self.name = name
self.signal = Signal(name)
self.event_type = event_type
self.is_sent_to_interfaces = send_to_interfaces
self.message = message
def send(self, sender, **kwargs):
"""Sends the signal with data
Args:
sender: The thing that is sending the signal
Kwargs:
data: Additional data to send with the signal
"""
self.signal.send(sender, **kwargs)
def connect(self, func, weak=True):
"""A decorator which registers a function as a callback for this signal
Args:
func (func): The function to register
weak (bool, optional): Should a weak reference be used for this connection? Defaults to True
Returns:
func: The function connected
"""
self.signal.connect(func)
if not weak:
WalkoffSignal._store_callback(func)
return func
@classmethod
def _store_callback(cls, func):
"""
Stores callbacks so they aren't garbage collected and the weak references of the signals disappear
"""
cls._signals[id(func)] = func
class WalkoffSignal(object):
"""A signal to send Walkoff data
The class is a wrapper around a blinker.Signal
Attributes:
name (str): The name of the signal
signal (Signal): The signal object which sends the event and data
event_type (EventType): The event type of this signal
is_sent_to_interfaces (bool, optional): Should this event get sent to the interface dispatcher? Defaults to True
message (str): Human readable message for this event
Args:
name (str): The name of the signal
event_type (EventType): The event type of this signal
send_to_interfaces (bool, optional): Should this event get sent to the interface dispatcher? Defaults to True
message (str, optional): Human readable message for this event. Defaults to empty string
"""
_signals = {}
def __init__(self, name, event_type, send_to_interfaces=True, message=''):
self.name = name
self.signal = Signal(name)
self.event_type = event_type
self.is_sent_to_interfaces = send_to_interfaces
self.message = message
def send(self, sender, **kwargs):
"""Sends the signal with data
Args:
sender: The thing that is sending the signal
Kwargs:
data: Additional data to send with the signal
"""
self.signal.send(sender, **kwargs)
def connect(self, func, weak=True):
"""A decorator which registers a function as a callback for this signal
Args:
func (func): The function to register
weak (bool, optional): Should a weak reference be used for this connection? Defaults to True
Returns:
func: The function connected
"""
self.signal.connect(func)
if not weak:
WalkoffSignal._store_callback(func)
return func
@classmethod
def _store_callback(cls, func):
"""
Stores callbacks so they aren't garbage collected and the weak references of the signals disappear
"""
cls._signals[id(func)] = func
class Event:
def __init__(self, callback, name=''):
self.signal = Signal()
self.name = name
self.callback = callback(
name
) # needed b/c blinker cannot use weak references for callbacks
self.signal.connect(self.callback)
def send(self, sender):
self.signal.send(sender)
def test_temp_connection_for_sender():
sig = Signal()
canary = []
receiver = lambda sender: canary.append(sender)
with sig.connected_to(receiver, sender=2):
sig.send(1)
sig.send(2)
assert canary == [2]
assert not sig.receivers
def anonymous_signal():
"""
匿名信号的例子
:return: no return
"""
# 定义一个信号
animal_signal = Signal()
# 信号注册一个接收者
animal_signal.connect(animal)
# 发送信号
animal_signal.send("anonymous")
class Settings():
"""Make multiple *Value instances accessible as a dict with their name as key"""
def __init__(self, *values):
# _values and _values_dict hold the same instances; the list is to
# preserve order and the dict is for fast access via __getitem__
self._values = []
self._values_dict = {}
self._on_change = Signal()
self.load(*values)
def add(self, value):
"""Add `value` to collection"""
self._values.append(value)
self._values_dict[value.name] = value
def load(self, *values):
"""Add multiple `values` to collection"""
for v in values:
self.add(v)
@property
def values(self):
"""Iterate over collected values"""
yield from self._values
@property
def names(self):
"""Iterate over values' `name` properties"""
yield from self._values_dict.keys()
def on_change(self, callback, autoremove=True):
"""
Run `callback` every time a value changes
`callback` gets the value instances as the only argument.
If `autoremove` is True, stop calling `callback` once it is garbage
collected.
"""
self._on_change.connect(callback, weak=autoremove)
def __getitem__(self, name):
return self._values_dict[name]
def __setitem__(self, name, value):
self._values_dict[name].value = value
self._on_change.send(self._values_dict[name])
def __contains__(self, name):
return name in self._values_dict
def go(self, signal: Signal, *args, **kwargs):
"""
在对应的时期点触发信号
如果此信号没有订阅者 那么将不会被真正发送
:param signal: 信号
:param args: 参数
:param kwargs: 字典参数
:return: none
"""
if signal is None:
raise ValueError("signal can not be null")
has_receivers = bool(signal.receivers)
if has_receivers:
try:
signal.send(*args, **kwargs)
except Exception as e:
pass
class IngestionFeeder(object):
def __init__(self, version):
"""
:param version: the version the records produced by this feeder will be
"""
self.version = version
self.read_signal = Signal(
doc=u'''Signal fired for each record read from the feeder. The
kwargs passed when the signal is triggered are "number"
and "record", the number is the number of the record from
the feeder so far (so essentially a count) and the record
is the actual record object.''')
self.finish_signal = Signal(
doc=u'''Signal fired when the feeder has been exhausted and all
records read. One kwarg is passed when the signal is
triggered: "number", the total number of records read.
''')
@property
@abc.abstractmethod
def source(self):
return None
@abc.abstractmethod
def records(self):
"""
Abstract function which when iterated over produces records. This could therefore either
return an iterable type (like a list, or set) or yield results as a generator (the latter is
recommended). An example implementation of this function: a csv parser that yields each row
until the CSV is exhausted.
:return: an iterable or yields each record
"""
return []
def read(self):
"""
Generator function which yields each record from the source.
"""
number = 0
for number, record in enumerate(self.records(), start=1):
self.read_signal.send(self, number=number, record=record)
yield record
self.finish_signal.send(self, number=number)
class Settings():
"""Dict of ordered Value objects"""
def __init__(self, *values):
# _values and _values_dict hold the same instances; the list is to
# preserve order and the dict is for access via __getitem__, etc.
self._values = []
self._values_dict = {}
self._on_change = Signal()
self.load(*values)
def add(self, value):
self._values.append(value)
self._values_dict[value.name] = value
def load(self, *values):
for v in values:
self.add(v)
@property
def names(self):
yield from self._values_dict.keys()
@property
def values(self):
yield from self._values
def on_change(self, callback, autoremove=True):
"""Run `callback` every time a value changes with the value
If `autoremove` is True, stop calling callback once it is garbage
collected.
"""
self._on_change.connect(callback, weak=autoremove)
def __getitem__(self, name):
return self._values_dict[name]
def __setitem__(self, name, value):
self._values_dict[name].set(value)
self._on_change.send(self._values_dict[name])
def __contains__(self, name):
return name in self._values_dict
class imageDisplay(NodeBase):
def __init__(self, name):
super(imageDisplay, self).__init__(name)
self.loadImage = Signal(str)
self.inExec = self.createInputPin(DEFAULT_IN_EXEC_NAME, 'ExecPin',
None, self.compute)
self.entity = self.createInputPin('path', 'StringPin')
self.outExec = self.createOutputPin(DEFAULT_OUT_EXEC_NAME, 'ExecPin',
None)
self.failed = self.createOutputPin("failed", 'ExecPin', None)
@staticmethod
def pinTypeHints():
helper = NodePinsSuggestionsHelper()
helper.addInputDataType('ExecPin')
helper.addInputDataType('StringPin')
helper.addInputStruct(PinStructure.Single)
return helper
@staticmethod
def category():
return 'UI'
@staticmethod
def keywords():
return ['image']
@staticmethod
def description():
return "Loads image to node body. This is UI only node"
def compute(self, *args, **kwargs):
path = self.entity.getData()
if os.path.exists(path):
self.loadImage.send(path)
self.outExec.call()
else:
self.failed.call()
class MailboxesAccountItem(NSObject):
def init(self):
self = super(MailboxesAccountItem, self).init()
self.folder_items = []
self.updated = Signal()
return self
@classmethod
def newBlank(cls):
return cls.alloc().init()
@classmethod
def newWithAccount_(cls, account):
self = cls.alloc().init()
self.account = account
account_updated.connect(objc_callback(self.folders_updated), account)
self.folders_updated(account)
return self
def folders_updated(self, account):
self.folder_items[:] = [MailboxesFolderItem.newWithFolder_(f)
for n, f in sorted(account._folders.items())]
self.updated.send(self)
class cons_d2c(object):
'''包含约束的d2c类'''
def __init__(self, d, constrains, init=True):
'''
Parameters:
d: 一个字典
constrains: 约束条目，每个元素都为一个三元元组（deps, func, target），例如，
(
(('k1.k11','k1.k12'), (lambda v1,v2:v1+v2), 'k2' ),
(('k3','k4'), (lambda v1,v2:(v1,v2)), ('k5','k6')),
)
init: bool, 是否使用约束条件初始化传入的字典
'''
self.__d2c_instance = d2c(d)
self.sig = Signal()
self.constrains = constrains
self.subscribers = self.__init_cons(constrains, init=init)
def __init_cons(self, constrains, init=True):
subscribers = []
for deps, func, target in constrains:
subs = _subscrib(deps, func, target, self)
subscribers.append(subs)
for _k in deps:
self.sig.connect(subs, sender=_k)
if init:
deps_set = set()
for deps, func, target in constrains:
deps_set.update(deps)
for dep in deps_set:
self.sig.send(dep)
return subscribers
def set(self, k, v, check=True):
_changed = self.__d2c_instance.set(k, v, check=check)
if _changed:
self.sig.send(k)
return _changed
def get(self, k):
return self.__d2c_instance.get(k)
def update(self, d, check=True):
k_changed = []
for k, v in d.items():
_changed = self.__d2c_instance.set(k, v, check=check)
if _changed:
k_changed.append(k)
for _k in k_changed:
self.sig.send(_k)
def __getattr__(self, item):
return getattr(self.__d2c_instance, item)
def test_temp_connection_alias():
sig = Signal()
canary = []
receiver = lambda sender: canary.append(sender)
sig.send(1)
with sig.temporarily_connected_to(receiver):
sig.send(2)
sig.send(3)
assert canary == [2]
assert not sig.receivers
def test_temp_connection_failure():
sig = Signal()
canary = []
receiver = lambda sender: canary.append(sender)
class Failure(Exception):
pass
try:
sig.send(1)
with sig.connected_to(receiver):
sig.send(2)
raise Failure
sig.send(3)
except Failure:
pass
else:
raise AssertionError("Context manager did not propagate.")
assert canary == [2]
assert not sig.receivers
class Session(object):
def __init__(self,
username,
password,
account_id,
auth_token=None,
user_directory='mpx',
region='US1',
service_registry=None,
token_duration=43200000, # 12 hours
token_idle_timeout=14400000, # 4 hours
use_ssl=True,
):
self.username = username
self.password = password
self.account = account_id
self.auth_token = auth_token
self.user_directory = user_directory
self.region = region
self.token_duration = token_duration
self.token_idle_timeout = token_idle_timeout
self.use_ssl = use_ssl
self.registry_url = REGISTRY_URL.format(tld=self.regional_tld)
self.signin_url = SIGN_IN_URL.format(tld=self.regional_tld)
self.post_sign_in = Signal()
self._registry = service_registry
self.session = requests.Session()
self.session.mount('https://', TLS1Adapter())
self.session.headers.update({
'Content-Type': 'application/json',
'Accept': 'application/json',
'User-Agent': 'Python Mediaamp %s' % __version__,
})
@property
def registry(self):
if self._registry is None:
self._registry = self.resolve_domain()
return self._registry
@property
def regional_tld(self):
return 'eu' if 'eu' in self.region.lower() else 'com'
@property
def signin_username(self):
return self.user_directory + '/' + self.username
def resolve_domain(self):
resp = self.get(self.registry_url, params={
'schema': '1.1',
'_accountId': self.account,
})
try:
return resp['resolveDomainResponse']
except KeyError:
raise MediaAmpError('Unexpected response loading registry.')
def sign_in(self):
self.auth_token = None
self.session.auth = HTTPBasicAuth(self.signin_username, self.password)
result = self.get(self.signin_url, is_signin_request=True, params={
'schema': '1.0',
'_duration': self.token_duration,
'_idleTimeout': self.token_idle_timeout,
})
try:
self.auth_token = result['signInResponse']['token']
except KeyError:
raise AuthenticationError('Could not retrieve token.')
self.post_sign_in.send(self)
def request_json(self, method, url, retry_sign_in=True, is_signin_request=False, **kwargs):
""" Requests JSON content from the supplied URL.
This is the primary function to be used to make requests to the MPX API.
Not only does it ensure that the body of the response can be encoded as
Python via JSON, it also wraps exceptions and will auto-login using
the supplied credentials when necessary (e.g. when a token expires).
This API is known to return 200 statuses for requests that fail. It's
their convention to include the HTTP response code in the body of
the JSON returned. This checks for that case and turns them into actual
exceptions.
"""
if self.auth_token is not None:
self.session.auth = HTTPBasicAuth('', self.auth_token)
elif not is_signin_request:
self.sign_in()
response = getattr(self.session, method)(url, **kwargs)
try:
response.raise_for_status()
except requests.HTTPError as e:
wrap_http_error(e)
try:
data = response.json() if response.text else {}
except ValueError:
raise MediaAmpError('Response body can not be read as JSON. ')
try:
raise_for_json_exception(data)
except InvalidTokenError:
if retry_sign_in:
self.sign_in()
return self.request_json(method, url, retry_sign_in=False, **kwargs)
else:
raise
return data
def get(self, url, **kwargs):
return self.request_json('get', url, **kwargs)
def put(self, url, **kwargs):
return self.request_json('put', url, **kwargs)
def post(self, url, **kwargs):
return self.request_json('post', url, **kwargs)
def delete(self, url, **kwargs):
return self.request_json('delete', url, **kwargs)
def __getitem__(self, key):
url = self.registry.get(key)
if url is None:
url = self.registry(key + ' read-only')
if url is None:
raise KeyError(key + ' not available.')
if self.use_ssl:
url = url.replace('http://', 'https://')
return services[key](self, url)
class ScriptRunner(object):
def __init__(
self,
session_id,
report,
enqueue_forward_msg,
client_state,
request_queue,
uploaded_file_mgr=None,
):
"""Initialize the ScriptRunner.
(The ScriptRunner won't start executing until start() is called.)
Parameters
----------
session_id : str
The ReportSession's id.
report : Report
The ReportSession's report.
client_state : streamlit.proto.ClientState_pb2.ClientState
The current state from the client (widgets and query params).
request_queue : ScriptRequestQueue
The queue that the ReportSession is publishing ScriptRequests to.
ScriptRunner will continue running until the queue is empty,
and then shut down.
uploaded_file_mgr : UploadedFileManager
The File manager to store the data uploaded by the file_uploader widget.
"""
self._session_id = session_id
self._report = report
self._enqueue_forward_msg = enqueue_forward_msg
self._request_queue = request_queue
self._uploaded_file_mgr = uploaded_file_mgr
self._client_state = client_state
self._widgets = Widgets()
self._widgets.set_state(client_state.widget_states)
self.on_event = Signal(doc="""Emitted when a ScriptRunnerEvent occurs.
This signal is *not* emitted on the same thread that the
ScriptRunner was created on.
Parameters
----------
event : ScriptRunnerEvent
exception : BaseException | None
Our compile error. Set only for the
SCRIPT_STOPPED_WITH_COMPILE_ERROR event.
widget_states : streamlit.proto.WidgetStates_pb2.WidgetStates | None
The ScriptRunner's final WidgetStates. Set only for the
SHUTDOWN event.
""")
# Set to true when we process a SHUTDOWN request
self._shutdown_requested = False
# Set to true while we're executing. Used by
# maybe_handle_execution_control_request.
self._execing = False
# This is initialized in start()
self._script_thread = None
def __repr__(self) -> str:
return util.repr_(self)
def start(self):
"""Start a new thread to process the ScriptEventQueue.
This must be called only once.
"""
if self._script_thread is not None:
raise Exception("ScriptRunner was already started")
self._script_thread = ReportThread(
session_id=self._session_id,
enqueue=self._enqueue_forward_msg,
query_string=self._client_state.query_string,
widgets=self._widgets,
uploaded_file_mgr=self._uploaded_file_mgr,
target=self._process_request_queue,
name="ScriptRunner.scriptThread",
)
self._script_thread.start()
def _process_request_queue(self):
"""Process the ScriptRequestQueue and then exits.
This is run in a separate thread.
"""
LOGGER.debug("Beginning script thread")
while not self._shutdown_requested and self._request_queue.has_request:
request, data = self._request_queue.dequeue()
if request == ScriptRequest.STOP:
LOGGER.debug("Ignoring STOP request while not running")
elif request == ScriptRequest.SHUTDOWN:
LOGGER.debug("Shutting down")
self._shutdown_requested = True
elif request == ScriptRequest.RERUN:
self._run_script(data)
else:
raise RuntimeError("Unrecognized ScriptRequest: %s" % request)
# Send a SHUTDOWN event before exiting. This includes the widget values
# as they existed after our last successful script run, which the
# ReportSession will pass on to the next ScriptRunner that gets
# created.
client_state = ClientState()
client_state.query_string = self._client_state.query_string
self._widgets.marshall(client_state)
self.on_event.send(ScriptRunnerEvent.SHUTDOWN,
client_state=client_state)
def _is_in_script_thread(self):
"""True if the calling function is running in the script thread"""
return self._script_thread == threading.current_thread()
def maybe_handle_execution_control_request(self):
if not self._is_in_script_thread():
# We can only handle execution_control_request if we're on the
# script execution thread. However, it's possible for deltas to
# be enqueued (and, therefore, for this function to be called)
# in separate threads, so we check for that here.
return
if not self._execing:
# If the _execing flag is not set, we're not actually inside
# an exec() call. This happens when our script exec() completes,
# we change our state to STOPPED, and a statechange-listener
# enqueues a new ForwardEvent
return
# Pop the next request from our queue.
request, data = self._request_queue.dequeue()
if request is None:
return
LOGGER.debug("Received ScriptRequest: %s", request)
if request == ScriptRequest.STOP:
raise StopException()
elif request == ScriptRequest.SHUTDOWN:
self._shutdown_requested = True
raise StopException()
elif request == ScriptRequest.RERUN:
raise RerunException(data)
else:
raise RuntimeError("Unrecognized ScriptRequest: %s" % request)
def _install_tracer(self):
"""Install function that runs before each line of the script."""
def trace_calls(frame, event, arg):
self.maybe_handle_execution_control_request()
return trace_calls
# Python interpreters are not required to implement sys.settrace.
if hasattr(sys, "settrace"):
sys.settrace(trace_calls)
@contextmanager
def _set_execing_flag(self):
"""A context for setting the ScriptRunner._execing flag.
Used by maybe_handle_execution_control_request to ensure that
we only handle requests while we're inside an exec() call
"""
if self._execing:
raise RuntimeError("Nested set_execing_flag call")
self._execing = True
try:
yield
finally:
self._execing = False
def _run_script(self, rerun_data):
"""Run our script.
Parameters
----------
rerun_data: RerunData
The RerunData to use.
"""
assert self._is_in_script_thread()
LOGGER.debug("Running script %s", rerun_data)
# Reset DeltaGenerators, widgets, media files.
media_file_manager.clear_session_files()
ctx = get_report_ctx()
if ctx is None:
# This should never be possible on the script_runner thread.
raise RuntimeError(
"ScriptRunner thread has a null ReportContext. Something has gone very wrong!"
)
ctx.reset(query_string=rerun_data.query_string)
self.on_event.send(ScriptRunnerEvent.SCRIPT_STARTED)
# Compile the script. Any errors thrown here will be surfaced
# to the user via a modal dialog in the frontend, and won't result
# in their previous report disappearing.
try:
with source_util.open_python_file(self._report.script_path) as f:
filebody = f.read()
if config.get_option("runner.magicEnabled"):
filebody = magic.add_magic(filebody, self._report.script_path)
code = compile(
filebody,
# Pass in the file path so it can show up in exceptions.
self._report.script_path,
# We're compiling entire blocks of Python, so we need "exec"
# mode (as opposed to "eval" or "single").
mode="exec",
# Don't inherit any flags or "future" statements.
flags=0,
dont_inherit=1,
# Use the default optimization options.
optimize=-1,
)
except BaseException as e:
# We got a compile error. Send an error event and bail immediately.
LOGGER.debug("Fatal script error: %s" % e)
self.on_event.send(
ScriptRunnerEvent.SCRIPT_STOPPED_WITH_COMPILE_ERROR,
exception=e)
return
# If we get here, we've successfully compiled our script. The next step
# is to run it. Errors thrown during execution will be shown to the
# user as ExceptionElements.
# Update the Widget object with the new widget_states.
# (The ReportContext has a reference to this object, so we just update it in-place)
if rerun_data.widget_states is not None:
self._widgets.set_state(rerun_data.widget_states)
if config.get_option("runner.installTracer"):
self._install_tracer()
# This will be set to a RerunData instance if our execution
# is interrupted by a RerunException.
rerun_with_data = None
try:
# Create fake module. This gives us a name global namespace to
# execute the code in.
module = _new_module("__main__")
# Install the fake module as the __main__ module. This allows
# the pickle module to work inside the user's code, since it now
# can know the module where the pickled objects stem from.
# IMPORTANT: This means we can't use "if __name__ == '__main__'" in
# our code, as it will point to the wrong module!!!
sys.modules["__main__"] = module
# Add special variables to the module's globals dict.
# Note: The following is a requirement for the CodeHasher to
# work correctly. The CodeHasher is scoped to
# files contained in the directory of __main__.__file__, which we
# assume is the main script directory.
module.__dict__["__file__"] = self._report.script_path
with modified_sys_path(self._report), self._set_execing_flag():
exec(code, module.__dict__)
except RerunException as e:
rerun_with_data = e.rerun_data
except StopException:
pass
except BaseException as e:
handle_uncaught_app_exception(e)
finally:
self._widgets.reset_triggers()
self._widgets.cull_nonexistent(ctx.widget_ids_this_run.items())
self.on_event.send(ScriptRunnerEvent.SCRIPT_STOPPED_WITH_SUCCESS)
# delete expired files now that the script has run and files in use
# are marked as active
media_file_manager.del_expired_files()
# Use _log_if_error() to make sure we never ever ever stop running the
# script without meaning to.
_log_if_error(_clean_problem_modules)
if rerun_with_data is not None:
self._run_script(rerun_with_data)
class PinBase(IPin):
"""
**Base class for pins**
This is the base class that stores the data in the graph.
This class is intended to be subclassed for each new registered data type you want to create.
:param _packageName: This holds the package where the sub classed pin is registered.
It is not intended to be set by developer, PyFlow automatically fills this property at
registration point
:type _packageName: str
Signals:
* **serializationHook** : Fired when Serialize Pin called, so Ui wrapers can append data to the serialization object
* **onPinConnected** : Fired when a new connection is made to this Pin, sends other Pin
* **onPinDisconnected** : Fired when some disconnection is made to this Pin, sends other Pin
* **nameChanged** : Fired when pin.setName() called, sends New Name
* **killed** : Fired when Pin gets deleted
* **onExecute** : Fired when Pin execution gets called
* **containerTypeChanged** : Fired when Pin Structure Changes
* **dataBeenSet** : Fired when data changes, sends New Data
* **dictChanged** : Fired when current structure changes to :py:const:`PyFlow.Core.Common.StructureType.Dict`, sends Dict key DataType
* **errorOccurred** : Fired when some error fired, like incorrect dataType set, sends ocurred Error
* **errorCleared** : Fired when error cleared
:ivar owningNode: Weak reference to owning node
:ivar reconnectionPolicy: What to do if connect with busy pin. Used when :attr:`~PyFlow.Core.Common.PinOptions.AllowMultipleConnections` flag is disabled
:ivar dirty: This flag for lazy evaluation
:ivar affects: List of pins this pin affects to
:ivar affected_by: List of pins that affects to this pin
:ivar name: Pin name
:ivar direction: Pin direction
:ivar inputWidgetVariant: Input widget variant tag
:ivar constraint: **description here**
:ivar structConstraint: **description here**
:ivar super: **description here**
:ivar activeDataType: Current data type of this pin. Used by AnyPin
:ivar pinIndex: Position of this pin on node
:ivar description: Text description of this pin
"""
_packageName = ""
def __init__(self, name, owningNode, direction):
super(PinBase, self).__init__()
# signals
self.serializationHook = Signal()
self.onPinConnected = Signal(object)
self.onPinDisconnected = Signal(object)
self.nameChanged = Signal(str)
self.killed = Signal()
self.onExecute = Signal(object)
self.containerTypeChanged = Signal()
self.dataBeenSet = Signal(object)
self.dictChanged = Signal(str)
self.markedAsDirty =Signal()
self.errorOccured = Signal(object)
self.errorCleared = Signal()
self._lastError = None
## Access to the node
self.owningNode = weakref.ref(owningNode)
self._uid = uuid.uuid4()
self._data = None
self._defaultValue = None
self.reconnectionPolicy = PinReconnectionPolicy.DisconnectIfHasConnections
self.dirty = True
self.affects = set()
self.affected_by = set()
self.name = name
self._group = ""
self.direction = direction
# gui class weak ref
self._wrapper = None
self.__wrapperJsonData = None
self.annotationDescriptionDict = None
self._inputWidgetVariant = "DefaultWidget"
# Constraint ports
self.constraint = None
self.structConstraint = None
# Flags
self._flags = PinOptions.Storable
self._origFlags = self._flags
self._structure = StructureType.Single
self._currStructure = self._structure
self._isAny = False
self._isArray = False
self._isDict = False
self._alwaysList = False
self._alwaysDict = False
self._alwaysSingle = False
self._defaultSupportedDataTypes = self._supportedDataTypes = self.supportedDataTypes()
self.canChange = False
self._isDictElement = False
self.hidden = False
# DataTypes
self.super = self.__class__
self.activeDataType = self.__class__.__name__
self._keyType = None
# registration
self.owningNode().pins.add(self)
self.owningNode().pinsCreationOrder[self.uid] = self
# This is for to be able to connect pins by location on node
self.pinIndex = 0
if direction == PinDirection.Input:
self.pinIndex = len(self.owningNode().orderedInputs)
if direction == PinDirection.Output:
self.pinIndex = len(self.owningNode().orderedOutputs)
self.description = "{} instance".format(self.dataType)
@property
def wrapperJsonData(self):
try:
dt = self.__wrapperJsonData.copy()
return dt
except Exception as e:
return None
def getInputWidgetVariant(self):
return self._inputWidgetVariant
def setInputWidgetVariant(self, value):
self._inputWidgetVariant = value
def path(self):
owningNodePath = self.owningNode().path()
return "{}.{}".format(owningNodePath, self.getName())
@property
def group(self):
"""Pin group
This is just a tag which can be used in ui level
:rtype: str
"""
return self._group
@group.setter
def group(self, value):
self._group = str(value)
def enableOptions(self, *options):
"""Enables flags on pin instance
Example:
>>> self.pinInstance.enableOptions(PinOptions.RenamingEnabled)
You can also pass array/set of flags
>>> self.pinInstance.enableOptions({PinOptions.RenamingEnabled, PinOptions.Dynamic})
This is equivalent of
>>> self.pinInstance.enableOptions(PinOptions.RenamingEnabled | PinOptions.Dynamic)
"""
for option in options:
self._flags = self._flags | option
self._origFlags = self._flags
def disableOptions(self, *options):
"""Same as :meth:`~PyFlow.Core.PinBase.PinBase.enableOptions` but inverse
"""
for option in options:
self._flags = self._flags & ~option
self._origFlags = self._flags
def optionEnabled(self, option):
"""Is option enabled or not
:param option: Option to check
:type option: :class:`~PyFlow.Core.Common.PinOptions`
:rtype: bool
"""
return bool(self._flags & option)
def isAny(self):
"""Wheter this pin of type Any or not
:rtype: bool
"""
return self._isAny
@property
def packageName(self):
"""Returns name of package this pin belongs to
:rtype: bool
"""
return self._packageName
@property
def linkedTo(self):
"""store connection from pins
from left hand side to right hand side
.. code-block:: python
{
"lhsNodeName": "", "outPinId": 0,
"rhsNodeName": "", "inPinId": 0
}
where pin id is order in which pin was added to node
:returns: Serialized connections
:rtype: list(dict)
"""
result = list()
if self.direction == PinDirection.Output:
for i in getConnectedPins(self):
connection = {"lhsNodeName": "", "outPinId": 0, "rhsNodeName": "", "inPinId": 0}
connection["lhsNodeName"] = self.owningNode().getName()
connection["lhsNodeUid"] = str(self.owningNode().uid)
connection["outPinId"] = self.pinIndex
connection["rhsNodeName"] = i.owningNode().getName()
connection["rhsNodeUid"] = str(i.owningNode().uid)
connection["inPinId"] = i.pinIndex
result.append(connection)
if self.direction == PinDirection.Input:
for i in getConnectedPins(self):
connection = {"lhsNodeName": "", "outPinId": 0, "rhsNodeName": "", "inPinId": 0}
connection["lhsNodeName"] = i.owningNode().getName()
connection["lhsNodeUid"] = str(i.owningNode().uid)
connection["outPinId"] = i.pinIndex
connection["rhsNodeName"] = self.owningNode().getName()
connection["rhsNodeUid"] = str(self.owningNode().uid)
connection["inPinId"] = self.pinIndex
result.append(connection)
return result
def __repr__(self):
return "[{0}:{1}:{2}:{3}]".format(self.dataType, self.getFullName(), self.dirty, self.currentData())
def isExec(self):
"""Returns whether this is exec pin or not
:rtype: bool
"""
return False
def initAsArray(self, bIsArray):
"""Sets this pins to be a list always
:param bIsArray: Define as array
:type bIsArray: bool
"""
self._alwaysList = bool(bIsArray)
if bool(bIsArray):
self._alwaysDict = False
self.setAsArray(bool(bIsArray))
def initAsDict(self, bIsDict):
"""Sets this pins to be a dict always
:param bIsArray: Define as dict
:type bIsArray: bool
"""
self._alwaysDict = bool(bIsDict)
if bool(bIsDict):
self._alwaysList = False
self.setAsDict(bool(bIsDict))
def setAsArray(self, bIsArray):
"""Sets this pins to be a list
:param bIsArray: Define as Array
:type bIsArray: bool
"""
bIsArray = bool(bIsArray)
if self._isArray == bIsArray:
return
self._isArray = bIsArray
if bIsArray:
if self.isDict():
self.setAsDict(False)
# list pins supports only lists by default
self.enableOptions(PinOptions.SupportsOnlyArrays)
self._currStructure = StructureType.Array
self._isDict = False
else:
self._currStructure = self._structure
self._data = self.defaultValue()
self.containerTypeChanged.send()
def setAsDict(self, bIsDict):
"""Sets this pins to be a dict
:param bIsArray: Define as Array
:type bIsArray: bool
"""
bIsDict = bool(bIsDict)
if self._isDict == bIsDict:
return
self._isDict = bIsDict
if bIsDict:
if self.isArray():
self.setAsArray(False)
# list pins supports only lists by default
self.enableOptions(PinOptions.SupportsOnlyArrays)
self._currStructure = StructureType.Dict
self._isArray = False
else:
self._currStructure = self._structure
self._keyType = None
self._data = self.defaultValue()
self.containerTypeChanged.send()
def isArray(self):
"""Returns whether this pin is array or not
:rtype: bool
"""
return self._isArray
def isDict(self):
"""Returns whether this pin is dict or not
:rtype: bool
"""
return self._isDict
def setWrapper(self, wrapper):
"""Sets ui wrapper instance
:param wrapper: Whatever ui class that represents this pin
"""
if self._wrapper is None:
self._wrapper = weakref.ref(wrapper)
def getWrapper(self):
"""Returns ui wrapper instance
"""
return self._wrapper
def deserialize(self, jsonData):
"""Restores itself from supplied serialized data
:param jsonData: Json representation of pin
:type jsonData: dict
"""
self.setName(jsonData["name"])
self.uid = uuid.UUID(jsonData['uuid'])
for opt in PinOptions:
if opt.value in jsonData["options"]:
self.enableOptions(opt)
else:
self.disableOptions(opt)
self.changeStructure(jsonData["structure"])
self._alwaysList = jsonData['alwaysList']
self._alwaysSingle = jsonData['alwaysSingle']
self._alwaysDict = jsonData['alwaysDict']
try:
self.setData(json.loads(jsonData['value'], cls=self.jsonDecoderClass()))
except Exception as e:
self.setData(self.defaultValue())
if "wrapper" in jsonData:
self.__wrapperJsonData = jsonData["wrapper"]
def serialize(self):
"""Serializes itself to json
:rtype: dict
"""
storable = self.optionEnabled(PinOptions.Storable)
serializedData = None
if not self.dataType == "AnyPin":
if storable:
serializedData = json.dumps(self.currentData(), cls=self.jsonEncoderClass())
#else:
# serializedData = json.dumps(self.defaultValue(), cls=self.jsonEncoderClass())
data = {
'name': self.name,
'package': self.packageName,
'fullName': self.getFullName(),
'dataType': self.__class__.__name__,
'direction': int(self.direction),
'value': serializedData,
'uuid': str(self.uid),
'linkedTo': list(self.linkedTo),
'pinIndex': self.pinIndex,
'options': [i.value for i in PinOptions if self.optionEnabled(i)],
'structure': int(self._currStructure),
'alwaysList': self._alwaysList,
'alwaysSingle': self._alwaysSingle,
'alwaysDict': self._alwaysDict
}
# Wrapper class can subscribe to this signal and return
# UI specific data which will be considered on serialization
# Blinker returns a tuple (receiver, return val)
wrapperData = self.serializationHook.send(self)
if wrapperData is not None:
if len(wrapperData) > 0:
# We take return value from one wrapper
data['wrapper'] = wrapperData[0][1]
return data
@property
def uid(self):
return self._uid
@uid.setter
def uid(self, value):
if not value == self._uid:
self._uid = value
def setName(self, name, force=False):
"""Sets pin name and fires events
:param name: New pin name
:type name: str
:param force: If True - name will be changed even if option :attr:`~PyFlow.Core.Common.PinOptions.RenamingEnabled` is turned off
:type force: bool
:returns: Whether renaming performed or not
:rtype: bool
"""
if not force:
if not self.optionEnabled(PinOptions.RenamingEnabled):
return False
if name == self.name:
return False
self.name = self.owningNode().getUniqPinName(name)
self.nameChanged.send(self.name)
return True
def getName(self):
return self.name
def getFullName(self):
"""Returns full pin name, including node name
:rtype: str
"""
return self.owningNode().name + '_' + self.name
def allowedDataTypes(self, checked=[], dataTypes=[], selfCheck=True, defaults=False):
return list(self.supportedDataTypes())
def checkFree(self, checked=[], selfCheck=True):
return False
def defaultValue(self):
"""Returns default value of this pin
"""
if self.isArray():
return []
elif self.isDict():
return PFDict("StringPin", "AnyPin")
else:
return self._defaultValue
def getData(self):
"""Returns pin value
If something is connected to this pin, graph will be evaluated
.. seealso:: :class:`~PyFlow.Core.EvaluationEngine.DefaultEvaluationEngine_Impl`
"""
return EvaluationEngine().getPinData(self)
def clearError(self):
"""Clears any last error on this pin and fires event
"""
if self._lastError is not None:
self._lastError = None
self.errorCleared.send()
def setError(self, err):
"""Marks this pin as invalid by setting error message to it. Also fires event
:param err: Error message
:type err: str
"""
self._lastError = str(err)
self.errorOccured.send(self._lastError)
def validateArray(self, array, func):
valid = True
if isinstance(array, list):
for i in array:
self.validateArray(i, func)
else:
func(array)
return valid
def setData(self, data):
"""Sets value to pin
:param data: Data to be set
:type data: object
"""
if self.super is None:
return
try:
self.setDirty()
if isinstance(data, DictElement) and not self.optionEnabled(PinOptions.DictElementSupported):
data = data[1]
if not self.isArray() and not self.isDict():
if isinstance(data, DictElement):
self._data = DictElement(data[0], self.super.processData(data[1]))
else:
if isinstance(data, list):
self._data = data
else:
self._data = self.super.processData(data)
elif self.isArray():
if isinstance(data, list):
if self.validateArray(data, self.super.processData):
self._data = data
else:
raise Exception("Some Array Input is not valid Data")
else:
self._data = [self.super.processData(data)]
elif self.isDict():
if isinstance(data, PFDict):
self._data = PFDict(data.keyType, data.valueType)
for key, value in data.items():
self._data[key] = self.super.processData(value)
elif isinstance(data, DictElement) and len(data) == 2:
self._data.clear()
self._data[data[0]] = self.super.processData(data[1])
if self.direction == PinDirection.Output:
for i in self.affects:
i.setData(self.currentData())
elif self.direction == PinDirection.Input and self.owningNode().__class__.__name__ == "compound":
for i in self.affects:
i.setData(self.currentData())
if self.direction == PinDirection.Input or self.optionEnabled(PinOptions.AlwaysPushDirty):
push(self)
self.clearError()
self.dataBeenSet.send(self)
except Exception as exc:
self.setError(exc)
self.setDirty()
if self._lastError is not None:
self.owningNode().setError(self._lastError)
wrapper = self.owningNode().getWrapper()
if wrapper:
wrapper.update()
def call(self, *args, **kwargs):
if self.owningNode().isValid():
self.onExecute.send(*args, **kwargs)
def disconnectAll(self):
if self.direction == PinDirection.Input:
for o in list(self.affected_by):
disconnectPins(self, o)
self.affected_by.clear()
if self.direction == PinDirection.Output:
for i in list(self.affects):
disconnectPins(self, i)
self.affects.clear()
@property
def dataType(self):
"""Returns data type of this pin
:rtype: str
"""
return self.__class__.__name__
@property
def structureType(self):
"""Returns current structure of this pin
:rtype: :class:`~PyFlow.Core.Common.StructureType`
"""
return self._structure
@structureType.setter
def structureType(self, structure):
self._structure = structure
self._currStructure = structure
# PinBase methods
def kill(self, *args, **kwargs):
"""Deletes this pin
"""
self.disconnectAll()
if self in self.owningNode().pins:
self.owningNode().pins.remove(self)
if self.uid in self.owningNode().pinsCreationOrder:
self.owningNode().pinsCreationOrder.pop(self.uid)
# Fix pin indexes on owning node
if self.optionEnabled(PinOptions.Dynamic):
# sort owning node pins indexes
index = 1
if self.direction == PinDirection.Input:
for inputPin in self.owningNode().orderedInputs.values():
if inputPin == self:
continue
inputPin.pinIndex = index
index += 1
index = 1
if self.direction == PinDirection.Output:
for outputPin in self.owningNode().orderedOutputs.values():
if outputPin == self:
continue
outputPin.pinIndex = index
index += 1
self.killed.send(self)
clearSignal(self.killed)
def currentData(self):
"""Returns current value of this pin, without any graph evaluation
:rtype: object
"""
if self._data is None:
return self._defaultValue
return self._data
def aboutToConnect(self, other):
"""This method called right before two pins connected
:param other: Pin which this pin is going to be connected with
:type other: :class:`~PyFlow.Core.PinBase.PinBase`
"""
if other.structureType != self.structureType:
if self.optionEnabled(PinOptions.ChangeTypeOnConnection) or self.structureType == StructureType.Multi:
self.changeStructure(other._currStructure)
self.onPinConnected.send(other)
def getCurrentStructure(self):
"""Returns this pin structure type
:rtype: :class:`~PyFlow.Core.Common.StructureType`
"""
if self.structureType == StructureType.Multi:
if self._alwaysSingle:
return StructureType.Single
elif self._alwaysList:
return StructureType.Array
elif self._alwaysDict:
return StructureType.Dict
else:
return self.structureType
else:
return self.structureType
def changeStructure(self, newStruct, init=False):
"""Changes this pin structure type
:param newStruct: Target structure
:type newStruct: :class:`~PyFlow.Core.Common.StructureType`
:param init: **docs goes here**
:type init: bool
"""
free = self.canChangeStructure(newStruct, [], init=init)
if free:
self.updateConstrainedPins(set(), newStruct, init, connecting=True)
def canChangeStructure(self, newStruct, checked=[], selfCheck=True, init=False):
"""Recursive function to determine if pin can change its structure
:param newStruct: New structure we want to apply
:type newStruct: string
:param checked: Already visited pins, defaults to []
:type checked: list, optional
:param selfCheck: Define if check pin itself for connected pins, defaults to True
:type selfCheck: bool, optional
:param init: Initialization flag, if set multi pins can became other structure and don't be able to change after new call with init=True, defaults to False
:type init: bool, optional
:returns: True if pin can change structure to newStruct
:rtype: bool
"""
if not init and (self._alwaysList or self._alwaysSingle or self._alwaysDict):
return False
if self.structConstraint is None or self.structureType == StructureType.Multi:
return True
elif self.structureType != StructureType.Multi:
return False
else:
con = []
if selfCheck:
free = not self.hasConnections()
if not free:
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
else:
free = True
checked.append(self)
free = True
if selfCheck:
def testfree():
free = False
for pin in getConnectedPins(self):
if pin._structure == StructureType.Multi:
free = True
else:
free = False
break
return free
if any([self._currStructure == StructureType.Single and newStruct == StructureType.Array and not self.optionEnabled(PinOptions.ArraySupported) and self.hasConnections(),
self._currStructure == StructureType.Single and newStruct == StructureType.Dict and not self.optionEnabled(PinOptions.DictSupported) and self.hasConnections(),
self._currStructure == StructureType.Array and newStruct == StructureType.Single and self.optionEnabled(PinOptions.SupportsOnlyArrays) and self.hasConnections(),
self._currStructure == StructureType.Dict and newStruct == StructureType.Single and self.optionEnabled(PinOptions.SupportsOnlyArrays) and self.hasConnections(),
self._currStructure == StructureType.Array and newStruct == StructureType.Dict and self.hasConnections(),
self._currStructure == StructureType.Dict and newStruct == StructureType.Array and self.hasConnections()]):
free = testfree()
if free:
for port in self.owningNode().structConstraints[self.structConstraint] + con:
if port not in checked:
checked.append(port)
free = port.canChangeStructure(newStruct, checked, True, init=init)
if not free:
break
return free
def updateConstrainedPins(self, traversed, newStruct, init=False, connecting=False):
nodePins = set()
if self.structConstraint is not None:
nodePins = set(self.owningNode().structConstraints[self.structConstraint])
else:
nodePins = set([self])
for connectedPin in getConnectedPins(self):
if connectedPin.structureType == StructureType.Multi:
if connectedPin.canChangeStructure(self._currStructure, init=init):
nodePins.add(connectedPin)
for neighbor in nodePins:
if neighbor not in traversed and neighbor.structureType == StructureType.Multi:
neighbor.setAsArray(newStruct == StructureType.Array)
neighbor.setAsDict(newStruct == StructureType.Dict)
if connecting:
if init:
neighbor._alwaysList = newStruct == StructureType.Array
neighbor._alwaysSingle = newStruct == StructureType.Single
neighbor._alwaysDict = newStruct == StructureType.Dict
neighbor._currStructure = newStruct
neighbor.disableOptions(PinOptions.ArraySupported)
neighbor.disableOptions(PinOptions.DictSupported)
if newStruct == StructureType.Array:
neighbor.enableOptions(PinOptions.ArraySupported)
elif newStruct == StructureType.Dict:
neighbor.enableOptions(PinOptions.DictSupported)
elif newStruct == StructureType.Multi:
neighbor.enableOptions(PinOptions.ArraySupported)
neighbor.enableOptions(PinOptions.DictSupported)
elif newStruct == StructureType.Single:
neighbor.disableOptions(PinOptions.SupportsOnlyArrays)
else:
neighbor._currStructure = neighbor._structure
neighbor._data = neighbor.defaultValue()
traversed.add(neighbor)
neighbor.setData(neighbor.defaultValue())
neighbor.updateConstrainedPins(traversed, newStruct, init, connecting=connecting)
def pinConnected(self, other):
push(self)
if self.isDict():
self.updateConnectedDicts([], self._data.keyType)
def pinDisconnected(self, other):
self.onPinDisconnected.send(other)
push(other)
def canChangeTypeOnConnection(self, checked=[], can=True, extraPins=[], selfCheck=True):
"""Recursive function to determine if pin can change its dataType
:param checked: Already visited pins, defaults to []
:type checked: list, optional
:param can: Variable Updated during iteration, defaults to True
:type can: bool, optional
:param extraPins: extra pins, non constrained or connected to this pin but that want to check also, defaults to []
:type extraPins: list, optional
:param selfCheck: Define if check pin itself for connected pins, defaults to True
:type selfCheck: bool, optional
:returns: True if pin can becabe other dataType
:rtype: bool
"""
if not self.optionEnabled(PinOptions.ChangeTypeOnConnection):
return False
con = []
neis = []
if selfCheck:
if self.hasConnections():
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
else:
checked.append(self)
if self.constraint:
neis = self.owningNode().constraints[self.constraint]
for port in neis + con + extraPins:
if port not in checked and can:
checked.append(port)
can = port.canChangeTypeOnConnection(checked, can, selfCheck=True)
return can
def getDictElementNode(self, checked=[], node=None):
"""Get the connected :py:class:`PyFlow.Packages.PyFlowBase.Nodes.makeDictElement.makeDictElement` to this pin recursively
:param checked: Currently visited pins, defaults to []
:type checked: list, optional
:param node: founded node, defaults to None
:rtype: :class:`~PyFlow.Core.NodeBase.NodeBase` or None
"""
if self.owningNode().__class__.__name__ == "makeDictElement":
return self.owningNode()
con = []
neis = []
if self.hasConnections() and self.direction == PinDirection.Input:
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
if self.constraint:
neis = self.owningNode().constraints[self.constraint]
for port in con + neis:
if port not in checked and node == None:
checked.append(port)
node = port.getDictElementNode(checked, node)
return node
def getDictNode(self, checked=[], node=None):
"""Get the connected :py:class:`PyFlow.Packages.PyFlowBase.Nodes.makeDict.makeDict` or
:py:class:`PyFlow.Packages.PyFlowBase.Nodes.makeAnyDict.makeAnyDict` to this pin recursively
:param checked: Currently visited pins, defaults to []
:type checked: list, optional
:param node: founded node, defaults to None
:returns: founded node or None if not found
"""
if self.owningNode().__class__.__name__ in ["makeDict", "makeAnyDict"]:
return self.owningNode()
con = []
neis = []
if self.hasConnections():
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
if self.constraint:
neis = self.owningNode().constraints[self.constraint]
for port in con + neis:
if port not in checked and node == None:
checked.append(port)
node = port.getDictNode(checked, node)
return node
def supportDictElement(self, checked=[], can=True, selfCheck=True):
"""Iterative functions that search in all connected pins to see if they support DictElement nodes.
:param checked: Already visited pins, defaults to []
:type checked: list, optional
:param can: this is the variable that will be actualized during the recursive function, defaults to False
:type can: bool, optional
:param selfCheck: Define if look itself or no, defaults to True
:type selfCheck: bool, optional
:returns: True if can connect DictElement nodes to this pin
:rtype: bool
"""
if not self.optionEnabled(PinOptions.DictElementSupported):
return False
con = []
neis = []
if selfCheck:
if self.hasConnections() and self.direction == PinDirection.Input:
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
else:
checked.append(self)
if self.constraint and self.owningNode().__class__.__name__ != "makeDictElement":
neis = self.owningNode().constraints[self.constraint]
for port in neis + con:
if port not in checked and can:
checked.append(port)
can = port.supportDictElement(checked, can, selfCheck=True)
return can
def supportOnlyDictElement(self, checked=[], can=False, selfCheck=True):
"""Iterative Functions that search in all connected pins to see if they support only DictElement nodes, this
is done for nodes like makeDict and simmilars.
:param checked: Already Visited Pins, defaults to []
:type checked: list, optional
:param can: this is the variable that will be actualized during the recursive function, defaults to False
:type can: bool, optional
:param selfCheck: Defines if look itself or no, defaults to True
:type selfCheck: bool, optional
:returns: True if can connect only DictElement and Dicts nodes to this Pin
:rtype: bool
"""
if self.isDict():
return True
con = []
neis = []
if selfCheck:
if self.hasConnections() and self.direction == PinDirection.Output:
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
else:
checked.append(self)
if self.constraint and self.owningNode().__class__.__name__ != "makeDictElement":
neis = self.owningNode().constraints[self.constraint]
for port in neis + con:
if port not in checked and not can:
checked.append(port)
can = port.supportOnlyDictElement(checked, can, selfCheck=True)
return can
def updateConnectedDicts(self, checked=[], keyType=None):
"""Iterate over connected dicts pins and DictElements pins updating key data type
:param checked: Already visited pins, defaults to []
:type checked: list, optional
:param keyType: KeyDataType to set, defaults to None
:type keyType: string, optional
"""
if not self.isDict():
return
con = []
neis = []
if self.hasConnections():
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
if self.constraint:
neis = self.owningNode().constraints[self.constraint]
for port in con + neis:
if port not in checked and port.isDict():
checked.append(port)
port._keyType = keyType
if port._data.keyType != keyType:
port._data = PFDict(keyType, port.dataType)
port.dictChanged.send(keyType)
if port.getWrapper():
port.getWrapper()().update()
port.updateConnectedDicts(checked, keyType)
def setClean(self):
"""Sets dirty flag to True
"""
self.dirty = False
#if self.direction == PinDirection.Output:
# for i in self.affects:
# i.dirty = False
def setDirty(self):
"""Sets dirty flag to True
"""
if self.isExec():
return
self.dirty = True
for i in self.affects:
i.dirty = True
self.markedAsDirty.send()
def hasConnections(self):
"""Return the number of connections this pin has
:rtype: int
"""
numConnections = 0
if self.direction == PinDirection.Input:
numConnections += len(self.affected_by)
elif self.direction == PinDirection.Output:
numConnections += len(self.affects)
return numConnections > 0
def setDefaultValue(self, val):
"""In python, all user-defined classes are mutable
So make sure to store separate copy of value
For example if this is a Matrix, default value will be changed each time data has been set in original Matrix
:param val: defaultValue
:type val: object
"""
self._defaultValue = copy(val)
def updateConstraint(self, constraint):
self.constraint = constraint
if constraint in self.owningNode().constraints:
self.owningNode().constraints[constraint].append(self)
else:
self.owningNode().constraints[constraint] = [self]
def updateStructConstraint(self, constraint):
self.structConstraint = constraint
if constraint in self.owningNode().structConstraints:
self.owningNode().structConstraints[constraint].append(self)
else:
self.owningNode().structConstraints[constraint] = [self]
@staticmethod
def IsValuePin():
"""Returns whether this pin is value pin or not
:rtype: bool
"""
return True
@staticmethod
def pinDataTypeHint():
"""Hint of what data type is this pin, as well as default value for this data type.
Used to easily find pin classes by type id.
:rtype: tuple(str, object)
:raises NotImplementedError: If not implemented
"""
raise NotImplementedError('pinDataTypeHint method of PinBase is not implemented')
@staticmethod
def supportedDataTypes():
return ()
@staticmethod
def jsonEncoderClass():
"""Returns json encoder class for this pin
"""
return json.JSONEncoder
@staticmethod
def jsonDecoderClass():
"""Returns json decoder class for this pin
"""
return json.JSONDecoder
class MotionStage(object): def __init__(self, axes, constraints = None): from Action import EmergencyStop from blinker import Signal self.axes_idx = dict() for i, axis in enumerate(axes): self.axes_idx[axis] = i self.axes = [axis for axis in axes] self.constraints = constraints self.abort_action = EmergencyStop(self.axes) self.onCycleStarted = Signal() self.onCycleFinished = Signal() self.onCycleAborted = Signal() self.onDestinationChanged = Signal() self.onRunning = Signal() self.onInitializing = Signal() self.onInitialized = Signal() self.onInitiatorMinus = Signal() self.onInitiatorPlus = Signal() self.onPositionChanged = Signal() for axis in self.axes: axis.onInitializing.connect(self.onInitializing_repeat) axis.onInitialized.connect(self.onInitialized_repeat) axis.onInitiatorMinus.connect(self.onInitiatorMinus_repeat) axis.onInitiatorPlus.connect(self.onInitiatorPlus_repeat) axis.onRunning.connect(self.onRunning_repeat) axis.onPosition.connect(self.onPosition_repeat) self.worker_thread = None self.active = False self.destination = None self.cycle_clear() self.update() def __del__(self): self.abort() def cycle_clear(self): import Queue self.action_queue = Queue.Queue() def cycle_add_action(self, action): if not self.action_queue: self.cycle_clear() self.action_queue.put(action) @property def running(self): return tuple([axis.running for axis in self.axes]) @property def initializing(self): return tuple([axis.initializing for axis in self.axes]) @property def initialized(self): return tuple([axis.initialized for axis in self.axes]) @property def initiator_minus(self): return tuple([axis.initiator_minus for axis in self.axes]) @property def initiator_plus(self): return tuple([axis.initiator_plus for axis in self.axes]) @property def position(self): return tuple([axis.position for axis in self.axes]) def onRunning_repeat(self, sender, running): self.onRunning.send(self, axis=self.axes_idx[sender], running=running) def onPosition_repeat(self, sender, position): self.onPositionChanged.send(self, axis=self.axes_idx[sender], position=position) def onInitializing_repeat(self, sender, initializing): self.onInitializing.send(self, axis=self.axes_idx[sender], initializing=initializing) def onInitialized_repeat(self, sender, initialized): self.onInitialized.send(self, axis=self.axes_idx[sender], initialized=initialized) def onInitiatorMinus_repeat(self, sender, active): self.onInitiatorMinus.send(self, axis=self.axes_idx[sender], active=active) def onInitiatorPlus_repeat(self, sender, active): self.onInitiatorPlus.send(self, axis=self.axes_idx[sender], active=active) def update(self): old_position = self.position for axis in self.axes: axis.update() def set_destination(self, destination): current_position = self.position if not self.destination: self.update() self.destination = current_position if isinstance(destination, list) or isinstance(destination, tuple): if len(destination) != len(self.axes): raise ValueError self.destination = tuple(destination) if isinstance(destination, dict): new_destination = list(self.destination) for k in destination: new_destination[k] = destination[k] self.destination = tuple(new_destination) for i,dest in enumerate(self.destination): self.onDestinationChanged.send(self, axis = i, destination = dest) speed = None if not None in current_position: delta = [abs(a-b) for a,b in zip(self.destination, current_position)] max_delta = max(delta) if max_delta > 0: speed = [float(d)/float(max_delta) for d in delta] from Action import GotoAbsolute self.cycle_clear() self.cycle_add_action(GotoAbsolute(self.axes, self.destination, speed)) def can_cycle_start(self): return True # FIXME: Add constraint tests here def cycle_start(self): import threading, weakref if self.active: return False # if not self.can_cycle_start(): # return False self.current_action = None self.active = True self.worker_thread = threading.Thread(target = MotionStage.cycleWorker, name = "MotionControl.cycleWorker", args=(weakref.proxy(self),)) self.worker_thread.daemon = True self.worker_thread.start() self.onCycleStarted.send(self) def abort(self): import threading self.active = False if isinstance(self.worker_thread, threading.Thread): self.worker_thread.join() def cycleWorker(ref): abort_action = ref.abort_action try: import time while True: if not ref.active: raise CycleAbort() ref.update() if not ref.current_action or ref.current_action.ended(): if ref.action_queue.empty(): break ref.current_action = ref.action_queue.get_nowait() ref.current_action.execute() while True: if not ref.active: raise CycleAbort() ref.update() if ref.current_action.ended(): break ref.action_queue.task_done() ref.onCycleFinished.send(ref) except CycleAbort: ref.abort_action.execute() ref.onCycleAborted.send(ref) finally: try: while not ref.action_queue.empty(): ref.action_queue.get_nowait() ref.action_queue.task_done() except: pass ref.active = False def reference(self): from Action import Initiate, GotoAbsolute self.destination = None self.cycle_clear() self.cycle_add_action(Initiate([self.axes[0]])) self.cycle_add_action(GotoAbsolute([self.axes[0]], [0])) self.cycle_start()
class GraphBase(ISerializable):
"""Data structure representing a nodes graph
:var graphManager: reference to graph manager
:vartype graphManager: :class:`~PyFlow.Core.GraphManager.GraphManager`
:var nameChanged: signal emitted after graph name was changed
:vartype nameChanged: :class:`~blinker.base.Signal`
:var categoryChanged: signal emitted after graph category was changed
:vartype categoryChanged: :class:`~blinker.base.Signal`
:var childGraphs: a set of child graphs
:vartype childGraphs: :class:`set`
:var nodes: nodes storage. Dictionary with :class:`uuid.UUID` as key and :class:`~PyFlow.Core.NodeBase.NodeBase` as value
:vartype nodes: :class:`dict`
:var uid: Unique identifier
:vartype uid: :class:`uuid.UUID`
.. py:method:: parentGraph
:property:
:getter: Returns a reference to parent graph or None if this graph is root
:setter: Sets new graph as new parent for this graph
.. py:method:: name
:property:
:getter: Returns graph name
:setter: Sets new graph name and fires signal
.. py:method:: category
:property:
:getter: Returns graph category
:setter: Sets new graph category and fires signal
.. py:method:: pins
:property:
:getter: Returns dictionary with :class:`uuid.UUID` as key and :class:`~PyFlow.Core.PinBase.PinBase` as value
:rtype: dict
"""
def __init__(self,
name,
manager,
parentGraph=None,
category='',
uid=None,
*args,
**kwargs):
super(GraphBase, self).__init__(*args, **kwargs)
self.graphManager = manager
self._isRoot = False
self.nameChanged = Signal(str)
self.categoryChanged = Signal(str)
self.__name = name
self.__category = category
self._parentGraph = None
self.childGraphs = set()
self.parentGraph = parentGraph
self._nodes = {}
self._vars = {}
self.uid = uuid.uuid4() if uid is None else uid
manager.add(self)
def setIsRoot(self, bIsRoot):
"""Sets this graph as root
.. warning:: Used internally
:param bIsRoot: -- Root or not
:type bIsRoot: :class:`bool`
"""
self._isRoot = bIsRoot
def isRoot(self):
"""Whether this graph is root or not
:rtype: :class:`bool`
"""
return self._isRoot
def getVars(self):
"""Returns this graph's variables storage
:returns: :class:`uuid.UUID` - :class:`~PyFlow.Core.NodeBase.NodeBase` dict
:rtype: :class:`dict`
"""
return self._vars
@property
def parentGraph(self):
return self._parentGraph
@parentGraph.setter
def parentGraph(self, newParentGraph):
if self.isRoot():
self._parentGraph = None
return
if newParentGraph is not None:
if self._parentGraph is not None:
# remove self from old parent's children set
if self in self._parentGraph.childGraphs:
self._parentGraph.childGraphs.remove(self)
# add self to new parent's children set
newParentGraph.childGraphs.add(self)
# update parent
self._parentGraph = newParentGraph
def depth(self):
"""Returns depth level of this graph
:rtype: int
"""
result = 1
parent = self._parentGraph
while parent is not None:
result += 1
parent = parent.parentGraph
return result
def getVarList(self):
"""return list of variables from active graph
:rtype: list(:class:`~PyFlow.Core.Variable.Variable`)
"""
result = list(self._vars.values())
parent = self._parentGraph
while parent is not None:
result += list(parent._vars.values())
parent = parent.parentGraph
return result
def serialize(self, *args, **kwargs):
"""Returns serialized representation of this graph
:rtype: dict
"""
result = {
'name':
self.name,
'category':
self.category,
'vars': [v.serialize() for v in self._vars.values()],
'nodes': [n.serialize() for n in self._nodes.values()],
'depth':
self.depth(),
'isRoot':
self.isRoot(),
'parentGraphName':
str(self._parentGraph.name)
if self._parentGraph is not None else str(None)
}
return result
def populateFromJson(self, jsonData):
"""Populates itself from serialized data
:param jsonData: serialized graph
:type jsonData: dict
"""
self.clear()
self.name = self.graphManager.getUniqGraphName(jsonData['name'])
self.category = jsonData['category']
self.setIsRoot(jsonData['isRoot'])
if self.isRoot():
self.name = "root"
# restore vars
for varJson in jsonData['vars']:
var = Variable.deserialize(self, varJson)
self._vars[var.uid] = var
# restore nodes
for nodeJson in jsonData['nodes']:
# check if variable getter or setter and pass variable
nodeArgs = ()
nodeKwargs = {}
if nodeJson['type'] in ('getVar', 'setVar'):
nodeKwargs['var'] = self._vars[uuid.UUID(nodeJson['varUid'])]
nodeJson['owningGraphName'] = self.name
node = getRawNodeInstance(nodeJson['type'],
packageName=nodeJson['package'],
libName=nodeJson['lib'],
*nodeArgs,
**nodeKwargs)
self.addNode(node, nodeJson)
# restore connection
for nodeJson in jsonData['nodes']:
for nodeOutputJson in nodeJson['outputs']:
for linkData in nodeOutputJson['linkedTo']:
try:
lhsNode = self._nodes[uuid.UUID(
linkData["lhsNodeUid"])]
except Exception as e:
lhsNode = self.findNode(linkData["lhsNodeName"])
try:
lhsPin = lhsNode.orderedOutputs[linkData["outPinId"]]
except Exception as e:
print("lhsPin not found {0}".format(str(linkData)))
continue
try:
rhsNode = self._nodes[uuid.UUID(
linkData["rhsNodeUid"])]
except Exception as e:
rhsNode = self.findNode(linkData["rhsNodeName"])
try:
rhsPin = rhsNode.orderedInputs[linkData["inPinId"]]
except Exception as e:
continue
if not arePinsConnected(lhsPin, rhsPin):
connected = connectPins(lhsPin, rhsPin)
# assert(connected is True), "Failed to restore connection"
if not connected:
print("Failed to restore connection", lhsPin,
rhsPin)
connectPins(lhsPin, rhsPin)
def remove(self):
"""Removes this graph as well as child graphs. Deepest graphs will be removed first
"""
# graphs should be removed from leafs to root
for childGraph in set(self.childGraphs):
childGraph.remove()
# remove itself
self.graphManager.removeGraph(self)
def clear(self):
"""Clears content of this graph as well as child graphs. Deepest graphs will be cleared first
"""
# graphs should be cleared from leafs to root
for childGraph in set(self.childGraphs):
childGraph.clear()
# clear itself
for node in list(self._nodes.values()):
node.kill()
self._nodes.clear()
for var in list(self._vars.values()):
self.killVariable(var)
self._vars.clear()
@property
def name(self):
return self.__name
@name.setter
def name(self, value):
value = str(value)
if self.__name != value:
self.__name = value
self.nameChanged.send(self.__name)
@property
def category(self):
return self.__category
@category.setter
def category(self, value):
self.__category = str(value)
self.categoryChanged.send(self.__category)
def Tick(self, deltaTime):
"""Executed periodically
:param deltaTime: Elapsed time since last tick
:type deltaTime: float
"""
for node in self._nodes.values():
node.Tick(deltaTime)
@property
def pins(self):
result = {}
for n in self.getNodesList():
for pin in tuple(n.inputs.values()) + tuple(n.outputs.values()):
result[pin.uid] = pin
return result
def createVariable(self,
dataType=str('AnyPin'),
accessLevel=AccessLevel.public,
uid=None,
name=str("var")):
"""Creates variable inside this graph scope
:param dataType: Variable data type
:type dataType: str
:param accessLevel: Variable access level
:type accessLevel: :class:`~PyFlow.Core.Common.AccessLevel`
:param uid: Variable unique identifier
:type uid: :class:`uuid.UUID`
:param name: Variable name
:type name: str
"""
name = self.graphManager.getUniqVariableName(name)
var = Variable(self,
getPinDefaultValueByType(dataType),
name,
dataType,
accessLevel=accessLevel,
uid=uid)
self._vars[var.uid] = var
return var
# TODO: add arguments to deal with references of this var
# disconnect pins or mark nodes invalid
def killVariable(self, var):
"""Removes variable from this graph
:param var: Variable to remove
:type var: :class:`~PyFlow.Core.Variable.Variable`
"""
assert (isinstance(var, Variable))
if var.uid in self._vars:
popped = self._vars.pop(var.uid)
popped.killed.send()
def getNodes(self):
"""Returns this graph's nodes storage
:rtype: dict(:class:`~PyFlow.Core.NodeBase.NodeBase`)
"""
return self._nodes
def getNodesList(self, classNameFilters=[]):
"""Returns this graph's nodes list
:rtype: list(:class:`~PyFlow.Core.NodeBase.NodeBase`)
"""
if len(classNameFilters) > 0:
return [
n for n in self._nodes.values()
if n.__class__.__name__ in classNameFilters
]
else:
return [n for n in self._nodes.values()]
def findNode(self, name):
"""Tries to find node by name
:param name: Node name
:type name: str or None
"""
for i in self._nodes.values():
if i.name == name:
return i
return None
def getNodesByClassName(self, className):
"""Returns a list of nodes filtered by class name
:param className: Class name of target nodes
:type className: str
:rtype: list(:class:`~PyFlow.Core.NodeBase.NodeBase`)
"""
nodes = []
for i in self.getNodesList():
if i.__class__.__name__ == className:
nodes.append(i)
return nodes
def findPinByUid(self, uid):
"""Tries to find pin by uuid
:param uid: Unique identifier
:type uid: :class:`~uuid.UUID`
:rtype: :class:`~PyFlow.Core.PinBase.PinBase` or None
"""
pin = None
if uid in self.pins:
pin = self.pins[uid]
return pin
def findPin(self, pinName):
"""Tries to find pin by name
:param pinName: String to search by
:type pinName: str
:rtype: :class:`~PyFlow.Core.PinBase.PinBase` or None
"""
result = None
for pin in self.pins.values():
if pinName == pin.getFullName():
result = pin
break
return result
def getInputNode(self):
"""Creates and adds to graph :class:`~PyFlow.Packages.Base.Nodes.graphNodes.graphInputs` node
pins on this node will be exposed on compound node as input pins
:rtype: :class:`~PyFlow.Core.NodeBase.NodeBase`
"""
node = getRawNodeInstance("graphInputs", "Base")
self.addNode(node)
return node
def getOutputNode(self):
"""Creates and adds to graph :class:`~PyFlow.Packages.Base.Nodes.graphNodes.graphOutputs` node.
pins on this node will be exposed on compound node as output pins
:rtype: :class:`~PyFlow.Core.NodeBase.NodeBase`
"""
node = getRawNodeInstance("graphOutputs", "Base")
self.addNode(node)
return node
def addNode(self, node, jsonTemplate=None):
"""Adds node to storage
:param node: Node to add
:type node: NodeBase
:param jsonTemplate: serialized representation of node. This used when graph deserialized to do custom stuff after node will be added.
:type jsonTemplate: dict
:rtype: bool
"""
from PyFlow.Core.PathsRegistry import PathsRegistry
assert (node is not None), "failed to add node, None is passed"
if node.uid in self._nodes:
return False
# Check if this node is variable get/set. Variables created in child graphs are not visible to parent ones
# Do not disrupt variable scope
if node.__class__.__name__ in ['getVar', 'setVar']:
var = self.graphManager.findVariableByUid(node.variableUid())
variableLocation = var.location()
if len(variableLocation) > len(self.location()):
return False
if len(variableLocation) == len(self.location()):
if Counter(variableLocation) != Counter(self.location()):
return False
node.graph = weakref.ref(self)
if jsonTemplate is not None:
jsonTemplate['name'] = self.graphManager.getUniqNodeName(
jsonTemplate['name'])
else:
node.setName(self.graphManager.getUniqNodeName(node.name))
self._nodes[node.uid] = node
node.postCreate(jsonTemplate)
PathsRegistry().rebuild()
return True
def location(self):
"""Returns path to current location in graph tree
Example:
>>> ["root", "compound1", "compound2"]
means:
>>> # root
>>> # |- compound
>>> # |- compound2
:rtype: list(str)
"""
result = [self.name]
parent = self._parentGraph
while parent is not None:
result.insert(0, parent.name)
parent = parent.parentGraph
return result
def count(self):
"""Returns number of nodes
:rtype: int
"""
return self._nodes.__len__()
def plot(self):
"""Prints graph to console. May be useful for debugging
"""
depth = self.depth()
prefix = "".join(['-'] * depth) if depth > 1 else ''
parentGraphString = str(
None) if self.parentGraph is None else self.parentGraph.name
print(prefix + "GRAPH:" + self.name +
", parent:{0}".format(parentGraphString))
assert (self not in self.childGraphs)
for child in self.childGraphs:
child.plot()
class ScriptRunner:
def __init__(
self,
session_id: str,
session_data: SessionData,
enqueue_forward_msg: Callable[[ForwardMsg], None],
client_state: ClientState,
request_queue: ScriptRequestQueue,
session_state: SessionState,
uploaded_file_mgr: UploadedFileManager,
):
"""Initialize the ScriptRunner.
(The ScriptRunner won't start executing until start() is called.)
Parameters
----------
session_id : str
The AppSession's id.
session_data : SessionData
The AppSession's session data.
enqueue_forward_msg : Callable
Function to call to send a ForwardMsg to the frontend.
(When not running a unit test, this will be the enqueue function
of the AppSession instance that created this ScriptRunner.)
client_state : ClientState
The current state from the client (widgets and query params).
request_queue : ScriptRequestQueue
The queue that the AppSession is publishing ScriptRequests to.
ScriptRunner will continue running until the queue is empty,
and then shut down.
uploaded_file_mgr : UploadedFileManager
The File manager to store the data uploaded by the file_uploader widget.
"""
self._session_id = session_id
self._session_data = session_data
self._enqueue_forward_msg = enqueue_forward_msg
self._request_queue = request_queue
self._uploaded_file_mgr = uploaded_file_mgr
self._client_state = client_state
self._session_state: SessionState = session_state
self._session_state.set_widgets_from_proto(client_state.widget_states)
self.on_event = Signal(doc="""Emitted when a ScriptRunnerEvent occurs.
This signal is *not* emitted on the same thread that the
ScriptRunner was created on.
Parameters
----------
sender: ScriptRunner
The sender of the event (this ScriptRunner).
event : ScriptRunnerEvent
exception : BaseException | None
Our compile error. Set only for the
SCRIPT_STOPPED_WITH_COMPILE_ERROR event.
widget_states : streamlit.proto.WidgetStates_pb2.WidgetStates | None
The ScriptRunner's final WidgetStates. Set only for the
SHUTDOWN event.
""")
# Set to true when we process a SHUTDOWN request
self._shutdown_requested = False
# Set to true while we're executing. Used by
# _maybe_handle_execution_control_request.
self._execing = False
# This is initialized in start()
self._script_thread: Optional[threading.Thread] = None
def __repr__(self) -> str:
return util.repr_(self)
def start(self) -> None:
"""Start a new thread to process the ScriptEventQueue.
This must be called only once.
"""
if self._script_thread is not None:
raise Exception("ScriptRunner was already started")
self._script_thread = threading.Thread(
target=self._run_script_thread,
name="ScriptRunner.scriptThread",
)
self._script_thread.start()
def _get_script_run_ctx(self) -> ScriptRunContext:
"""Get the ScriptRunContext for the current thread.
Returns
-------
ScriptRunContext
The ScriptRunContext for the current thread.
Raises
------
AssertionError
If called outside of a ScriptRunner thread.
RuntimeError
If there is no ScriptRunContext for the current thread.
"""
assert self._is_in_script_thread()
ctx = get_script_run_ctx()
if ctx is None:
# This should never be possible on the script_runner thread.
raise RuntimeError(
"ScriptRunner thread has a null ScriptRunContext. Something has gone very wrong!"
)
return ctx
def _run_script_thread(self) -> None:
"""The entry point for the script thread.
Processes the ScriptRequestQueue, which will at least contain the RERUN
request that will trigger the first script-run.
When the ScriptRequestQueue is empty, or when a SHUTDOWN request is
dequeued, this function will exit and its thread will terminate.
"""
assert self._is_in_script_thread()
LOGGER.debug("Beginning script thread")
# Create and attach the thread's ScriptRunContext
ctx = ScriptRunContext(
session_id=self._session_id,
enqueue=self._enqueue,
query_string=self._client_state.query_string,
session_state=self._session_state,
uploaded_file_mgr=self._uploaded_file_mgr,
)
add_script_run_ctx(threading.current_thread(), ctx)
while not self._shutdown_requested and self._request_queue.has_request:
request, data = self._request_queue.dequeue()
if request == ScriptRequest.STOP:
LOGGER.debug("Ignoring STOP request while not running")
elif request == ScriptRequest.SHUTDOWN:
LOGGER.debug("Shutting down")
self._shutdown_requested = True
elif request == ScriptRequest.RERUN:
self._run_script(data)
else:
raise RuntimeError("Unrecognized ScriptRequest: %s" % request)
# Send a SHUTDOWN event before exiting. This includes the widget values
# as they existed after our last successful script run, which the
# AppSession will pass on to the next ScriptRunner that gets
# created.
client_state = ClientState()
client_state.query_string = ctx.query_string
widget_states = self._session_state.as_widget_states()
client_state.widget_states.widgets.extend(widget_states)
self.on_event.send(self,
event=ScriptRunnerEvent.SHUTDOWN,
client_state=client_state)
def _is_in_script_thread(self) -> bool:
"""True if the calling function is running in the script thread"""
return self._script_thread == threading.current_thread()
def _enqueue(self, msg: ForwardMsg) -> None:
"""Enqueue a ForwardMsg to our browser queue.
This private function is called by ScriptRunContext only.
It may be called from the script thread OR the main thread.
"""
# Whenever we enqueue a ForwardMsg, we also handle any pending
# execution control request. This means that a script can be
# cleanly interrupted and stopped inside most `st.foo` calls.
#
# (If "runner.installTracer" is true, then we'll actually be
# handling these requests in a callback called after every Python
# instruction instead.)
if not config.get_option("runner.installTracer"):
self._maybe_handle_execution_control_request()
# Pass the message up to our associated AppSession.
self._enqueue_forward_msg(msg)
def _maybe_handle_execution_control_request(self) -> None:
if not self._is_in_script_thread():
# We can only handle execution_control_request if we're on the
# script execution thread. However, it's possible for deltas to
# be enqueued (and, therefore, for this function to be called)
# in separate threads, so we check for that here.
return
if not self._execing:
# If the _execing flag is not set, we're not actually inside
# an exec() call. This happens when our script exec() completes,
# we change our state to STOPPED, and a statechange-listener
# enqueues a new ForwardEvent
return
# Pop the next request from our queue.
request, data = self._request_queue.dequeue()
if request is None:
return
LOGGER.debug("Received ScriptRequest: %s", request)
if request == ScriptRequest.STOP:
raise StopException()
elif request == ScriptRequest.SHUTDOWN:
self._shutdown_requested = True
raise StopException()
elif request == ScriptRequest.RERUN:
raise RerunException(data)
else:
raise RuntimeError("Unrecognized ScriptRequest: %s" % request)
def _install_tracer(self) -> None:
"""Install function that runs before each line of the script."""
def trace_calls(frame, event, arg):
self._maybe_handle_execution_control_request()
return trace_calls
# Python interpreters are not required to implement sys.settrace.
if hasattr(sys, "settrace"):
sys.settrace(trace_calls)
@contextmanager
def _set_execing_flag(self):
"""A context for setting the ScriptRunner._execing flag.
Used by _maybe_handle_execution_control_request to ensure that
we only handle requests while we're inside an exec() call
"""
if self._execing:
raise RuntimeError("Nested set_execing_flag call")
self._execing = True
try:
yield
finally:
self._execing = False
def _run_script(self, rerun_data: RerunData) -> None:
"""Run our script.
Parameters
----------
rerun_data: RerunData
The RerunData to use.
"""
assert self._is_in_script_thread()
LOGGER.debug("Running script %s", rerun_data)
# Reset DeltaGenerators, widgets, media files.
in_memory_file_manager.clear_session_files()
ctx = self._get_script_run_ctx()
ctx.reset(query_string=rerun_data.query_string)
self.on_event.send(self, event=ScriptRunnerEvent.SCRIPT_STARTED)
# Compile the script. Any errors thrown here will be surfaced
# to the user via a modal dialog in the frontend, and won't result
# in their previous script elements disappearing.
try:
with source_util.open_python_file(
self._session_data.main_script_path) as f:
filebody = f.read()
if config.get_option("runner.magicEnabled"):
filebody = magic.add_magic(filebody,
self._session_data.main_script_path)
code = compile(
filebody,
# Pass in the file path so it can show up in exceptions.
self._session_data.main_script_path,
# We're compiling entire blocks of Python, so we need "exec"
# mode (as opposed to "eval" or "single").
mode="exec",
# Don't inherit any flags or "future" statements.
flags=0,
dont_inherit=1,
# Use the default optimization options.
optimize=-1,
)
except BaseException as e:
# We got a compile error. Send an error event and bail immediately.
LOGGER.debug("Fatal script error: %s" % e)
self._session_state[SCRIPT_RUN_WITHOUT_ERRORS_KEY] = False
self.on_event.send(
self,
event=ScriptRunnerEvent.SCRIPT_STOPPED_WITH_COMPILE_ERROR,
exception=e,
)
return
# If we get here, we've successfully compiled our script. The next step
# is to run it. Errors thrown during execution will be shown to the
# user as ExceptionElements.
if config.get_option("runner.installTracer"):
self._install_tracer()
# This will be set to a RerunData instance if our execution
# is interrupted by a RerunException.
rerun_with_data = None
try:
# Create fake module. This gives us a name global namespace to
# execute the code in.
module = _new_module("__main__")
# Install the fake module as the __main__ module. This allows
# the pickle module to work inside the user's code, since it now
# can know the module where the pickled objects stem from.
# IMPORTANT: This means we can't use "if __name__ == '__main__'" in
# our code, as it will point to the wrong module!!!
sys.modules["__main__"] = module
# Add special variables to the module's globals dict.
# Note: The following is a requirement for the CodeHasher to
# work correctly. The CodeHasher is scoped to
# files contained in the directory of __main__.__file__, which we
# assume is the main script directory.
module.__dict__["__file__"] = self._session_data.main_script_path
with modified_sys_path(
self._session_data), self._set_execing_flag():
# Run callbacks for widgets whose values have changed.
if rerun_data.widget_states is not None:
# Update the WidgetManager with the new widget_states.
# The old states, used to skip callbacks if values
# haven't changed, are also preserved in the
# WidgetManager.
self._session_state.compact_state()
self._session_state.set_widgets_from_proto(
rerun_data.widget_states)
self._session_state.call_callbacks()
ctx.on_script_start()
exec(code, module.__dict__)
self._session_state[SCRIPT_RUN_WITHOUT_ERRORS_KEY] = True
except RerunException as e:
rerun_with_data = e.rerun_data
except StopException:
pass
except BaseException as e:
self._session_state[SCRIPT_RUN_WITHOUT_ERRORS_KEY] = False
handle_uncaught_app_exception(e)
finally:
self._on_script_finished(ctx)
# Use _log_if_error() to make sure we never ever ever stop running the
# script without meaning to.
_log_if_error(_clean_problem_modules)
if rerun_with_data is not None:
self._run_script(rerun_with_data)
def _on_script_finished(self, ctx: ScriptRunContext) -> None:
"""Called when our script finishes executing, even if it finished
early with an exception. We perform post-run cleanup here.
"""
self._session_state.reset_triggers()
self._session_state.cull_nonexistent(ctx.widget_ids_this_run)
# Signal that the script has finished. (We use SCRIPT_STOPPED_WITH_SUCCESS
# even if we were stopped with an exception.)
self.on_event.send(self,
event=ScriptRunnerEvent.SCRIPT_STOPPED_WITH_SUCCESS)
# Delete expired files now that the script has run and files in use
# are marked as active.
in_memory_file_manager.del_expired_files()
# Force garbage collection to run, to help avoid memory use building up
# This is usually not an issue, but sometimes GC takes time to kick in and
# causes apps to go over resource limits, and forcing it to run between
# script runs is low cost, since we aren't doing much work anyway.
if config.get_option("runner.postScriptGC"):
gc.collect(2)
class MotionControl(object): def __init__(self, axes, constraints = None): import Queue self.axes = axes self.constraints = constraints self.action_queue = Queue.Queue() self.abort_action = NullAction() self.onCycleStarted = Signal() self.onCycleFinished = Signal() self.onCycleAborted = Signal() self.active = False self.target = None def __del__(self): self.abort() def __getattr__(self, name): if name == 'position': return [axis.position for axis in self.axes] def update(self): for axis in self.axes: axis.update() def set_target(self, target): if isinstance(target, list): if len(target) != len(self.axes): raise ValueError self.target = target if isinstance(target, dict): for k,v in target: self.target[k] = v if isinstance(target, tuple): self.target[target[0]] = target[1] speed = None current_position = self.position if not None in current_position: delta = [abs(a-b) for a,b in zip(target, current_position)] max_delta = max(delta) speed = [float(d)/float(max_delta) for d in delta] self.action_queue = Queue.Queue() self.action_queue.put(GotoAbsolute(self.axes, self.target, speed)) def can_cycle_start(self): if self.active: return False return True # FIXME: Add constraint tests here def start_cycle(self): import threading, weakref if not self.can_cycle_start(): return False self.current_action = None self.active = True self.worker_thread = threading.Thread(target = MotionControl.cycle_worker, name = "MotionControl.worker", args=(weakref.proxy(self),)) self.worker_thread.daemon =True self.worker_thread.start() self.onCycleStarted.send() def abort(self): self.active = False self.worker_thread.join() def __del__(self): self.abort() def cycle_worker(ref): abort_action = ref.abort_action try: import time while True: if not ref.active: raise CycleAbort() ref.update() if not ref.current_action or ref.current_action.ended(): if ref.action_queue.empty(): break ref.current_action = ref.action_queue.get_nowait() ref.current_action.execute() while True: if not ref.active: raise CycleAbort() ref.update() if ref.current_action.ended(): break ref.action_queue.task_done() ref.onCycleFinished.send() except CycleAbort: ref.abort_action.execute() ref.onCycleAborted.send() finally: try: while not ref.action_queue.empty(): ref.action_queue.get_nowait() ref.action_queue.task_done() except: pass ref.active = False
class WokEngine(Synchronizable):
"""
The Wok engine manages the execution of workflow cases.
Each case represents a workflow loaded with a certain configuration.
"""
def __init__(self, conf, conf_base_path=None):
Synchronizable.__init__(self)
self._global_conf = conf
self._expanded_global_conf = conf.clone().expand_vars()
self._conf = self._expanded_global_conf.get("wok", default=Data.element)
self._conf_base_path = conf_base_path
self._log = logger.get_logger("wok.engine")
self._work_path = self._conf.get("work_path", os.path.join(os.getcwd(), "wok-files"))
if not os.path.exists(self._work_path):
os.makedirs(self._work_path)
self._cases = []
self._cases_by_name = {}
self._stopping_cases = {}
#self._lock = Lock()
self._cvar = threading.Condition(self._lock)
self._run_thread = None
self._running = False
self._finished_event = threading.Event()
self._job_task_map = {}
self._logs_threads = []
self._logs_queue = Queue()
self._join_thread = None
self._join_queue = Queue()
self._num_log_threads = self._conf.get("num_log_threads", cpu_count())
self._max_alive_threads = 2 + self._num_log_threads
self._num_alive_threads = AtomicCounter()
self._started = False
self._notified = False
recover = self._conf.get("recover", False)
db_path = os.path.join(self._work_path, "engine.db")
if not recover and os.path.exists(db_path):
os.remove(db_path)
self._db = db.create_engine("sqlite:///{}".format(db_path), drop_tables=not recover)
# platforms
self._platforms = self._create_platforms()
self._platforms_by_name = {}
for platform in self._platforms:
self._platforms_by_name[platform.name] = platform
default_platform_name = self._conf.get("default_platform", self._platforms[0].name)
if default_platform_name not in self._platforms_by_name:
self._log.warn("Platform '{}' not found, using '{}' as the default platform".format(
default_platform_name, self._platforms[0].name))
default_platform_name = self._platforms[0].name
self._default_platform = self._platforms_by_name[default_platform_name]
# projects
if conf_base_path is None:
conf_base_path = os.getcwd()
projects_conf = self._global_conf.get("wok.projects")
self._projects = ProjectManager(projects_conf, base_path=conf_base_path)
self._projects.initialize()
# signals
self.case_created = Signal()
self.case_state_changed = Signal()
self.case_started = Signal()
self.case_finished = Signal()
self.case_removed = Signal()
# recovering
if recover:
self.__recover_from_db()
def _create_platforms(self):
"""
Creates the platform according to the configuration
:return: Platform
"""
platform_confs = self._conf.get("platforms")
if platform_confs is None:
platform_confs = Data.list()
elif not Data.is_list(platform_confs):
self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs))
platform_confs = Data.list()
if len(platform_confs) == 0:
platform_confs += [Data.element(dict(type="local"))]
platforms = []
names = {}
for pidx, platform_conf in enumerate(platform_confs):
if isinstance(platform_conf, basestring):
if not os.path.isabs(platform_conf) and self._conf_base_path is not None:
platform_conf = os.path.join(self._conf_base_path, platform_conf)
platform_conf = ConfigLoader(platform_conf).load()
if not Data.is_element(platform_conf):
raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf))
ptype = platform_conf.get("type", "local")
name = platform_conf.get("name", ptype)
if name in names:
name = "{}-{}".format(name, names[name])
names[name] += 1
else:
names[name] = 2
platform_conf["name"] = name
if "work_path" not in platform_conf:
platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name))
self._log.info("Creating '{}' platform ...".format(name))
self._log.debug("Platform configuration: {}".format(repr(platform_conf)))
platforms += [create_platform(ptype, platform_conf)]
return platforms
def _on_job_update(self, event, **kwargs):
self.notify()
def __recover_from_db(self):
raise NotImplementedError()
def __queue_adaptative_get(self, queue, start_timeout=1.0, max_timeout=6.0):
timeout = start_timeout
msg = None
while self._running and msg is None:
try:
msg = queue.get(timeout=timeout)
except Empty:
if timeout < max_timeout:
timeout += 0.5
except:
break
return msg
# Not used anywhere
def __queue_batch_get(self, queue, start_timeout=1, max_timeout=5):
timeout = start_timeout
msg_batch = []
while self._running and len(msg_batch) == 0:
try:
msg_batch += [queue.get(timeout=timeout)]
while not queue.empty():
msg_batch += [queue.get(timeout=timeout)]
except Empty:
if timeout < max_timeout:
timeout += 1
return msg_batch
def __job_submissions(self, session, platform):
#FIXME Be fair with priorities between different cases ?
query = session.query(db.WorkItem)\
.filter(db.WorkItem.state == runstates.READY)\
.filter(db.WorkItem.platform == platform.name)\
.order_by(db.WorkItem.priority)
for workitem in query:
case = self._cases_by_name[workitem.case.name]
task = case.component(workitem.task.cname)
js = JobSubmission(
case=case,
task=task,
workitem_id=workitem.id,
job_name=workitem.cname,
task_conf=task.conf,
priority=workitem.priority)
execution = task.execution
cmd_builder = create_command_builder(execution.mode)
js.script, js.env = cmd_builder.prepare(case, task, workitem.index)
yield js
def __remove_case(self, session, case):
"""
Definitively remove a case. The engine should be locked and no case jobs running.
"""
self._log.info("Dropping case {} ...".format(case.name))
del self._cases_by_name[case.name]
self._cases.remove(case)
# remove engine db objects and finalize case
self._log.debug(" * database ...")
case.remove(session)
self._lock.release()
try:
#TODO clean the job manager output files
try:
self._log.debug(" * logs ...")
logs_path = os.path.join(self._work_path, "logs", case.name)
shutil.rmtree(logs_path)
except:
self._log.exception("Error removing logs at {}".format(logs_path))
# remove data
self._log.debug(" * data ...")
for platform in case.platforms:
platform.data.remove_case(case.name)
# remove storage
self._log.debug(" * storage ...")
for platform in case.platforms:
platform.storage.delete_container(case.name)
# emit signal
self.case_removed.send(case)
finally:
self._lock.acquire()
# threads ----------------------
@synchronized
def _run(self):
set_thread_title()
num_exc = 0
self._running = True
self._num_alive_threads += 1
# Start the logs threads
for i in range(self._num_log_threads):
t = threading.Thread(target=self._logs, args=(i, ), name="wok-engine-logs-%d" % i)
self._logs_threads += [t]
t.start()
# Start the join thread
self._join_thread = threading.Thread(target=self._join, name="wok-engine-join")
self._join_thread.start()
_log = logger.get_logger("wok.engine.run")
_log.debug("Engine run thread ready")
while self._running:
session = db.Session()
try:
#_log.debug("Scheduling new tasks ...")
set_thread_title("scheduling")
updated_tasks = set()
# schedule tasks ready to be executed and save new workitems into the db
for case in self._cases:
tasks = case.schedule(session)
updated_tasks.update(tasks)
session.commit()
# submit workitems ready to be executed
for platform in self._platforms:
job_submissions = self.__job_submissions(session, platform)
for js, job_id, job_state in platform.submit(job_submissions):
workitem = session.query(db.WorkItem).filter(db.WorkItem.id == js.workitem_id).one()
workitem.job_id = job_id
workitem.state = job_state
js.task.dirty = True
session.commit()
updated_tasks.add(js.task)
session.close()
session = None
#_log.debug("Waiting for events ...")
set_thread_title("waiting")
while len(updated_tasks) == 0 and not self._notified and self._running:
self._cvar.wait(1)
self._notified = False
if not self._running:
break
session = db.Session() # there is a session.close() in the finished block
#_log.debug("Stopping jobs for aborting instances ...")
set_thread_title("working")
# check stopping instances
for case in self._cases:
if (case.state == runstates.ABORTING or case.removed) and case not in self._stopping_cases:
num_job_ids = session.query(db.WorkItem.job_id).filter(db.WorkItem.case_id == case.id)\
.filter(~db.WorkItem.state.in_(runstates.TERMINAL_STATES)).count()
if num_job_ids == 0:
if case.state == runstates.ABORTING:
_log.debug("Aborted case {} with no running jobs".format(case.name))
dbcase = session.query(db.Case).filter(db.Case.id == case.id)
dbcase.state = case.state = runstates.ABORTED
session.commit()
else:
_log.debug("Stopped case {} with no running jobs".format(case.name))
if case.removed:
_log.debug("Removing case {} with no running jobs".format(case.name))
self.__remove_case(session, case)
session.commit()
else:
_log.info("Stopping {} jobs for case {} ...".format(num_job_ids, case.name))
self._stopping_cases[case] = set()
for platform in self._platforms:
job_ids = [int(r[0]) for r in session.query(db.WorkItem.job_id)
.filter(db.WorkItem.case_id == case.id)\
.filter(db.WorkItem.platform == platform.name)\
.filter(~db.WorkItem.state.in_(runstates.TERMINAL_STATES))]
self._stopping_cases[case].update(job_ids)
platform.jobs.abort(job_ids)
#_log.debug("Checking job state changes ...")
# detect workitems which state has changed
for platform in self._platforms:
for job_id, state in platform.jobs.state():
try:
workitem = session.query(db.WorkItem).filter(db.WorkItem.job_id == job_id).one()
except NoResultFound:
_log.warn("No work-item available for the job {0} while retrieving state".format(job_id))
platform.jobs.abort([job_id])
platform.jobs.join(job_id)
continue
if workitem.state != state:
case = self._cases_by_name[workitem.case.name]
task = case.component(workitem.task.cname)
task.dirty = True
workitem.state = state
workitem.substate = runstates.LOGS_RETRIEVAL
session.commit()
updated_tasks.add(task)
# if workitem has finished, queue it for logs retrieval
if state in runstates.TERMINAL_STATES:
self._logs_queue.put((workitem.id, job_id))
_log.debug("[{}] Work-Item {} changed state to {}".format(case.name, workitem.cname, state))
#_log.debug("Updating components state ...")
# update affected components state
updated_cases = set([task.case for task in updated_tasks])
for case in updated_cases:
case.update_states(session)
case.update_count_by_state(session)
case.clean_components(session)
session.commit()
if case.state == runstates.RUNNING:
self._lock.release()
try:
self.case_started.send(case)
finally:
self._lock.acquire()
for task in updated_tasks:
case = task.case
#_log.debug("[{}] Component {} updated state to {} ...".format(
# component.case.name, component.cname, component.state))
count = task.workitem_count_by_state
sb = ["[{}] {} ({})".format(case.name, task.cname, task.state.title)]
sep = " "
for state in runstates.STATES:
if state in count:
sb += [sep, "{}={}".format(state.symbol, count[state])]
if sep == " ":
sep = ", "
if task.state == runstates.FINISHED and task.state in count:
elapsed = str(task.elapsed)
elapsed = elapsed.split(".")[0]
sb += [" ", "<{}>".format(elapsed)]
self._log.info("".join(sb))
except BaseException as ex:
num_exc += 1
_log.warn("Exception in run thread ({}): {}".format(num_exc, str(ex)))
#if num_exc > 3:
# raise
#else:
from traceback import format_exc
_log.debug(format_exc())
try:
if session is not None:
session.rollback()
except Exception as ex:
_log.warn("Session rollback failed")
_log.exception(ex)
finally:
try:
if session is not None:
session.close()
except Exception as ex:
_log.warn("Session close failed")
_log.exception(ex)
session = None
set_thread_title("finishing")
try:
# print cases state before leaving the thread
#for case in self._cases:
# _log.debug("Case state:\n" + repr(case))
for t in self._logs_threads:
t.join()
self._lock.release()
self._join_thread.join()
self._lock.acquire()
_log.debug("Engine run thread finished")
except Exception as ex:
_log.exception(ex)
self._running = False
self._num_alive_threads -= 1
def _logs(self, index):
"Log retrieval thread"
set_thread_title()
self._num_alive_threads += 1
_log = logger.get_logger("wok.engine.logs-{}".format(index))
_log.debug("Engine logs thread ready")
num_exc = 0
while self._running:
set_thread_title("waiting")
# get the next task to retrieve the logs
job_info = self.__queue_adaptative_get(self._logs_queue)
if job_info is None:
continue
workitem_id, job_id = job_info
session = db.Session()
task = None
try:
workitem = session.query(db.WorkItem).filter(db.WorkItem.id == workitem_id).one()
case = self._cases_by_name[workitem.case.name]
task = case.component(workitem.task.cname)
set_thread_title(workitem.cname)
_log.debug("[{}] Reading logs for work-item {} ...".format(case.name, workitem.cname))
output = task.platform.jobs.output(job_id)
if output is None:
output = StringIO.StringIO()
path = os.path.join(self._work_path, "logs", case.name, task.cname)
if not os.path.isdir(path):
try:
os.makedirs(path)
except:
if not os.path.isdir(path):
raise
path = os.path.join(path, "{:08}.db".format(workitem.index))
if os.path.isfile(path):
os.remove(path)
logs_db = LogsDb(path)
logs_db.open()
logs_db.add(case.name, task.cname, workitem.index, output)
logs_db.close()
_log.debug("[{}] Done with logs of work-item {}".format(case.name, workitem.cname))
except BaseException as ex:
num_exc += 1
session.rollback()
_log.info("Exception in logs thread ({}): {}".format(num_exc, str(ex)))
from traceback import format_exc
_log.debug(format_exc())
finally:
workitem.substate = runstates.JOINING
self._join_queue.put(job_info)
session.commit()
session.close()
self._num_alive_threads -= 1
_log.debug("Engine logs thread finished")
def _join(self):
"Joiner thread"
set_thread_title()
self._num_alive_threads += 1
_log = logger.get_logger("wok.engine.join")
_log.debug("Engine join thread ready")
session = None
num_exc = 0
while self._running:
try:
set_thread_title("waiting")
job_info = self.__queue_adaptative_get(self._join_queue)
if job_info is None:
continue
workitem_id, job_id = job_info
with self._lock:
session = db.Session()
workitem = session.query(db.WorkItem).filter(db.WorkItem.id == workitem_id).one()
case = self._cases_by_name[workitem.case.name]
task = case.component(workitem.task.cname)
set_thread_title(task.cname)
#_log.debug("Joining work-item %s ..." % task.cname)
jr = task.platform.jobs.join(job_id)
wr = Data.element(dict(
hostname=jr.hostname,
created=jr.created.strftime(_DT_FORMAT) if jr.created is not None else None,
started=jr.started.strftime(_DT_FORMAT) if jr.started is not None else None,
finished=jr.finished.strftime(_DT_FORMAT) if jr.finished is not None else None,
exitcode=jr.exitcode.code if jr.exitcode is not None else None))
r = task.platform.data.load_workitem_result(case.name, task.cname, workitem.index)
if r is not None:
if r.exception is not None:
wr["exception"] = r.exception
if r.trace is not None:
wr["trace"] = r.trace
workitem.substate = None
workitem.result = wr
case.num_active_workitems -= 1
session.commit()
# check if there are still more work-items
num_workitems = session.query(func.count(db.WorkItem.id)).filter(
~db.WorkItem.state.in_(runstates.TERMINAL_STATES)).scalar()
if self._single_run and num_workitems == 0:
stop_engine = True
for case in self._cases:
stop_engine = stop_engine and (case.state in runstates.TERMINAL_STATES)
#self._running = not stop_engine
if stop_engine:
self._finished_event.set()
_log.debug("[{}] Joined work-item {}".format(case.name, workitem.cname))
# check stopping instances
if case in self._stopping_cases:
job_ids = self._stopping_cases[case]
if job_id in job_ids:
job_ids.remove(job_id)
if len(job_ids) == 0:
del self._stopping_cases[case]
if case.state == runstates.ABORTING:
workitem.case.state = case.state = runstates.ABORTED
session.commit()
if case.removed:
self.__remove_case(session, case)
session.commit()
else:
_log.debug("Still waiting for {} jobs to stop".format(len(job_ids)))
if case.state in runstates.TERMINAL_STATES and case.num_active_workitems == 0:
_log.info("[{}] Case {}. Total time: {}".format(case.name, case.state.title, str(case.elapsed)))
self._lock.release()
try:
self.case_finished.send(case)
finally:
self._lock.acquire()
except BaseException as ex:
num_exc += 1
_log.warn("Exception in join thread ({}): {}".format(num_exc, str(ex)))
from traceback import format_exc
_log.debug(format_exc())
try:
if session is not None:
session.rollback()
except Exception as ex:
_log.warn("Session rollback failed")
_log.exception(ex)
finally:
try:
if session is not None:
session.close()
except Exception as ex:
_log.warn("Session close failed")
_log.exception(ex)
self._num_alive_threads -= 1
_log.debug("Engine join thread finished")
# API -----------------------------------
@property
def conf(self):
return self._conf
@property
def work_path(self):
return self._work_path
@property
def projects(self):
return self._projects
def platform(self, name):
return self._platforms_by_name.get(name)
@property
def default_platform(self):
return self._default_platform
@synchronized
def start(self, wait=True, single_run=False):
self._log.info("Starting engine ...")
started_platforms = []
try:
for platform in self._platforms:
started_platforms += [platform]
platform.start()
platform.callbacks.add(events.JOB_UPDATE, self._on_job_update)
except BaseException as ex:
self._log.error(str(ex))
for platform in started_platforms:
platform.close()
raise
#for project in self._projects:
# self._default_platform.sync_project(project)
self._single_run = single_run
self._run_thread = threading.Thread(target=self._run, name="wok-engine-run")
self._run_thread.start()
self._lock.release()
try:
try:
self._num_alive_threads.wait_condition(lambda value: value < self._max_alive_threads)
self._started = True
self._log.info("Engine started")
except KeyboardInterrupt:
wait = False
self._log.warn("Ctrl-C pressed ...")
except Exception as e:
wait = False
self._log.error("Exception while waiting for the engine to start")
self._log.exception(e)
if wait:
self.wait()
finally:
self._lock.acquire()
def wait(self):
self._log.info("Waiting for the engine to finish ...")
try:
finished = self._finished_event.wait(1)
while not finished:
finished = self._finished_event.wait(1)
except KeyboardInterrupt:
self._log.warn("Ctrl-C pressed ...")
except Exception as e:
self._log.error("Exception while waiting for the engine to finish, stopping the engine ...")
self._log.exception(e)
self._log.info("Finished waiting for the engine ...")
def _stop_threads(self):
self._log.info("Stopping threads ...")
if self._run_thread is not None:
with self._lock:
self._running = False
self._cvar.notify()
while self._run_thread.isAlive():
try:
self._run_thread.join(1)
except KeyboardInterrupt:
self._log.warn("Ctrl-C pressed, killing the process ...")
import signal
os.kill(os.getpid(), signal.SIGTERM)
except Exception as e:
self._log.error("Exception while waiting for threads to finish ...")
self._log.exception(e)
self._log.warn("killing the process ...")
exit(-1)
import signal
os.kill(os.getpid(), signal.SIGTERM)
self._run_thread = None
self._log.info("All threads finished ...")
@synchronized
def stop(self):
self._log.info("Stopping the engine ...")
self._finished_event.set()
self._lock.release()
try:
if self._run_thread is not None:
self._stop_threads()
for platform in self._platforms:
platform.close()
finally:
self._lock.acquire()
self._started = False
self._log.info("Engine stopped")
def running(self):
return self._started
def notify(self, lock=True):
if lock:
self._lock.acquire()
self._notified = True
self._cvar.notify()
if lock:
self._lock.release()
@synchronized
def cases(self):
instances = []
for inst in self._cases:
instances += [SynchronizedCase(self, inst)]
return instances
@synchronized
def case(self, name):
inst = self._cases_by_name.get(name)
if inst is None:
return None
return SynchronizedCase(self, inst)
@synchronized
def exists_case(self, name):
return name in self._cases_by_name
@synchronized
def create_case(self, case_name, conf_builder, project_name, flow_name, container_name):
"Creates a new workflow case"
session = db.Session()
if session.query(db.Case).filter(db.Case.name==case_name).count() > 0:
raise Exception("A case with this name already exists: {}".format(case_name))
flow_uri = "{}:{}".format(project_name, flow_name)
self._log.info("Creating case {} from {} ...".format(case_name, flow_uri))
try:
try:
flow = self.projects.load_flow(flow_uri)
project = flow.project
except:
self._log.error("Error while loading the workflow from {}".format(flow_uri))
raise
for platform in self._platforms:
try:
platform.data.remove_case(case_name)
platform.data.create_case(case_name)
except:
self._log.error("Error while initializing data for case {}".format(case_name))
raise
try:
case = Case(case_name, conf_builder, project, flow, container_name, engine=self)
self._cases += [case]
self._cases_by_name[case_name] = case
case.persist(session)
session.flush()
self.notify(lock=False)
except:
self._log.error("Error while creating case {} for the workflow {} with configuration {}".format(
case_name, flow_uri, conf_builder.get_conf()))
raise
except:
session.rollback()
#self._log.error("Error while creating case {} for the workflow {} with configuration {}".format(
# case_name, flow_uri, conf_builder.get_conf()))
raise
session.close()
self._log.debug("\n" + repr(case))
self._lock.release()
try:
self.case_created.send(case)
finally:
self._lock.acquire()
return SynchronizedCase(self, case)
@synchronized
def remove_case(self, name):
if name in self._cases_by_name:
session = db.Session()
case = self._cases_by_name[name]
dbcase = session.query(db.Case).filter(db.Case.id == case.id).first()
dbcase.removed = case.removed = True
if case.state not in runstates.TERMINAL_STATES + [runstates.READY]:
dbcase.state = case.state = runstates.ABORTING
num_retries = 3
while num_retries > 0:
try:
session.commit()
self.notify(lock=False)
self._log.debug("Case {} marked for removal".format(case.name))
except BaseException as ex:
num_retries -= 1
_log.info("Exception in remove_case: {}".format(str(ex)))
if num_retries > 0:
_log.info("Remaining retries = {}".format(num_retries))
import time
time.sleep(1)
else:
from traceback import format_exc
_log.debug(format_exc())
session.rollback()
finally:
session.close()
else:
self._log.error("Trying to remove a non existing case: {}".format(name))
'''
class NodeBase(INode):
_packageName = ""
def __init__(self, name, uid=None):
super(NodeBase, self).__init__()
self.bCacheEnabled = False
self.cacheMaxSize = 1000
self.cache = {}
self.killed = Signal()
self.tick = Signal(float)
self.errorOccured = Signal(object)
self.errorCleared = Signal()
self._uid = uuid.uuid4() if uid is None else uid
self.graph = None
self.name = name
self.pinsCreationOrder = OrderedDict()
self._pins = set()
self.x = 0.0
self.y = 0.0
self.bCallable = False
self._wrapper = None
self._constraints = {}
self._structConstraints = {}
self.lib = None
self.isCompoundNode = False
self._lastError = None
self.__wrapperJsonData = None
self._nodeMetaData = None
self.headerColor = None
self._deprecated = False
self._deprecationMessage = "This node is deprecated"
self._experimental = False
def setDeprecated(self, message):
self._deprecated = True
self._deprecationMessage = "This node will be removed in later releases! {}".format(
message)
def isDeprecated(self):
return self._deprecated
def isExperimental(self):
return self._experimental
def setExperimental(self):
self._experimental = True
def deprecationMessage(self):
return self._deprecationMessage
def getMetaData(self):
"""Returns node metadata dict.
This will return dict only for function based nodes. For class based nodes it will return None
.. seealso:: :mod:`~PyFlow.Core.FunctionLibrary`
:rtype: dict or None
"""
return self._nodeMetaData
@property
def wrapperJsonData(self):
try:
dt = self.__wrapperJsonData.copy()
self.__wrapperJsonData.clear()
self.__wrapperJsonData = None
return dt
except:
return None
def isValid(self):
return self._lastError is None
def clearError(self):
self._lastError = None
self.errorCleared.send()
def setError(self, err):
self._lastError = str(err)
self.errorOccured.send(self._lastError)
def checkForErrors(self):
failedPins = {}
for pin in self._pins:
if pin._lastError is not None:
failedPins[pin.name] = pin._lastError
if len(failedPins):
self._lastError = "Error on Pins:%s" % str(failedPins)
else:
self.clearError()
wrapper = self.getWrapper()
if wrapper:
wrapper.update()
@property
def packageName(self):
return self._packageName
@property
def constraints(self):
return self._constraints
@property
def structConstraints(self):
return self._structConstraints
def getOrderedPins(self):
return self.pinsCreationOrder.values()
def getter(self, pinName):
pin = self.getPinByName(pinName)
if not pin:
raise Exception()
else:
return pin
def __getitem__(self, pinName):
try:
return self.getter(pinName)
except Exception as x:
if "<str>" in str(x):
try:
return self.getter(str(pinName))
except:
raise Exception(
"Could not find pin with name:{0}".format(pinName))
else:
raise Exception(
"Could not find signature for __getitem__:{0}".format(
type(pinName)))
@property
def pins(self):
return self._pins
@property
def inputs(self):
"""Returns all input pins. Dictionary generated every time property called, so cache it when possible.
"""
result = OrderedDict()
for pin in self.pins:
if pin.direction == PinDirection.Input:
result[pin.uid] = pin
return result
@property
def orderedInputs(self):
result = {}
sortedInputs = sorted(self.inputs.values(), key=lambda x: x.pinIndex)
for inp in sortedInputs:
result[inp.pinIndex] = inp
return result
@property
def namePinInputsMap(self):
"""Returns all input pins. Dictionary generated every time property called, so cache it when possible.
"""
result = OrderedDict()
for pin in self.pins:
if pin.direction == PinDirection.Input:
result[pin.name] = pin
return result
@property
def outputs(self):
"""Returns all output pins. Dictionary generated every time property called, so cache it when possible.
"""
result = OrderedDict()
for pin in self.pins:
if pin.direction == PinDirection.Output:
result[pin.uid] = pin
return result
@property
def orderedOutputs(self):
result = {}
sortedOutputs = sorted(self.outputs.values(), key=lambda x: x.pinIndex)
for out in sortedOutputs:
result[out.pinIndex] = out
return result
@property
def namePinOutputsMap(self):
"""Returns all output pins. Dictionary generated every time property called, so cache it when possible.
"""
result = OrderedDict()
for pin in self.pins:
if pin.direction == PinDirection.Output:
result[pin.name] = pin
return result
# IItemBase interface
def setWrapper(self, wrapper):
if self._wrapper is None:
self._wrapper = wrapper
def getWrapper(self):
return self._wrapper
def location(self):
return self.graph().location()
def path(self):
location = "/".join(self.location())
return "{}/{}".format(location, self.getName())
@property
def uid(self):
return self._uid
@uid.setter
def uid(self, value):
if self.graph is not None:
self.graph().getNodes()[value] = self.graph().getNodes().pop(
self._uid)
self._uid = value
@staticmethod
def jsonTemplate():
template = {
'package': None,
'lib': None,
'type': None,
'owningGraphName': None,
'name': None,
'uuid': None,
'inputs': [],
'outputs': [],
'meta': {
'var': {}
},
'wrapper': {}
}
return template
def serialize(self):
template = NodeBase.jsonTemplate()
uidString = str(self.uid)
nodeName = self.name
template['package'] = self.packageName
template['lib'] = self.lib
template['type'] = self.__class__.__name__
template['name'] = nodeName
template['owningGraphName'] = self.graph().name
template['uuid'] = uidString
template['inputs'] = [i.serialize() for i in self.inputs.values()]
template['outputs'] = [o.serialize() for o in self.outputs.values()]
template['meta']['label'] = self.name
template['x'] = self.x
template['y'] = self.y
# if running with ui get ui wrapper data to save
wrapper = self.getWrapper()
if wrapper:
template['wrapper'] = wrapper.serializationHook()
return template
def isUnderActiveGraph(self):
return self.graph() == self.graph().graphManager.activeGraph()
def kill(self, *args, **kwargs):
from PyFlow.Core.PathsRegistry import PathsRegistry
if self.uid not in self.graph().getNodes():
return
self.killed.send()
for pin in self.inputs.values():
pin.kill()
for pin in self.outputs.values():
pin.kill()
self.graph().getNodes().pop(self.uid)
PathsRegistry().rebuild()
def Tick(self, delta):
self.tick.send(delta)
@staticmethod
def category():
return "Default"
@staticmethod
def keywords():
return []
@staticmethod
def pinTypeHints():
return NodePinsSuggestionsHelper()
@staticmethod
def description():
return "Default node description"
def getName(self):
return self.name
def setName(self, name):
self.name = str(name)
def useCache(self):
# if cached results exists - return them without calling compute
args = tuple([
pin.currentData() for pin in self.inputs.values()
if pin.IsValuePin()
])
# not hashable types will not be cached
for arg in args:
if not isinstance(arg, collections.Hashable):
return False
if args in self.cache:
for outPin, data in self.cache[args].items():
outPin.setData(data)
return True
def afterCompute(self):
if len(self.cache) >= self.cacheMaxSize:
return
# cache results
args = tuple([
pin.currentData() for pin in self.inputs.values()
if pin.IsValuePin()
])
for arg in args:
if not isinstance(arg, collections.Hashable):
return
cache = {}
for pin in self.outputs.values():
cache[pin] = pin.currentData()
self.cache[args] = cache
def processNode(self, *args, **kwargs):
if not self.isValid():
return
if self.bCacheEnabled:
if not self.useCache():
try:
self.compute()
self.clearError()
self.checkForErrors()
except Exception as e:
self.setError(e)
self.afterCompute()
else:
try:
self.compute()
self.clearError()
self.checkForErrors()
except Exception as e:
self.setError(e)
# INode interface
def compute(self, *args, **kwargs):
"""This is node's brains. Main logic goes here
Here are basic steps:
1. Get data from input pins
2. Do stuff
3. Set data to output pins
4. Call execs if needed
Here is compute method of charge node
.. code-block:: python
:linenos:
def compute(self, *args, **kwargs):
step = abs(self.step.getData())
if (self._currentAmount + step) < abs(self.amount.getData()):
self._currentAmount += step
return
self.completed.call(*args, **kwargs)
self._currentAmount = 0.0
.. note:: See :mod:`PyFlow.Packages.PyFlowBase.Nodes` source code module for more examples
"""
pass
# INode interface end
def isCallable(self):
"""Whether this node is callable or not
"""
for p in list(self.inputs.values()) + list(self.outputs.values()):
if p.isExec():
return True
return False
def setPosition(self, x, y):
"""Sets node coordinate on canvas
Used to correctly restore gui wrapper class
:param x: X coordinate
:type x: float
:param y: Y coordinate
:type y: float
"""
self.x = x
self.y = y
def autoAffectPins(self):
"""All value inputs affects on all value outputs. All exec inputs affects on all exec outputs
"""
for i in self.inputs.values():
for o in self.outputs.values():
assert (i is not o)
if not i.IsValuePin() and o.IsValuePin():
continue
if i.IsValuePin() and not o.IsValuePin():
continue
pinAffects(i, o)
def createInputPin(self,
pinName,
dataType,
defaultValue=None,
foo=None,
structure=PinStructure.Single,
constraint=None,
structConstraint=None,
supportedPinDataTypes=[],
group=""):
"""Creates input pin
:param pinName: Pin name
:type pinName: str
:param dataType: Pin data type
:type dataType: str
:param defaultValue: Pin default value
:type defaultValue: object
:param foo: Pin callback. used for exec pins
:type foo: function
:param structure: Pin structure
:type structure: :class:`~PyFlow.Core.Common.PinStructure.Single`
:param constraint: Pin constraint. Should be any hashable type. We use str
:type constraint: object
:param structConstraint: Pin struct constraint. Also should be hashable type
:type structConstraint: object
:param supportedPinDataTypes: List of allowed pin data types to be connected. Used by AnyPin
:type supportedPinDataTypes: list(str)
:param group: Pin group. Used only by ui wrapper
:type group: str
"""
pinName = self.getUniqPinName(pinName)
p = CreateRawPin(pinName, self, dataType, PinDirection.Input)
p.structureType = structure
p.group = group
if structure == PinStructure.Array:
p.initAsArray(True)
elif structure == PinStructure.Dict:
p.initAsDict(True)
elif structure == PinStructure.Multi:
p.enableOptions(PinOptions.ArraySupported)
if foo:
p.onExecute.connect(foo, weak=False)
if defaultValue is not None or dataType == "AnyPin":
p.setDefaultValue(defaultValue)
p.setData(defaultValue)
if dataType == "AnyPin":
p.setTypeFromData(defaultValue)
else:
p.setDefaultValue(getPinDefaultValueByType(dataType))
if dataType == "AnyPin" and supportedPinDataTypes:
def supportedDataTypes():
return supportedPinDataTypes
p._supportedDataTypes = p._defaultSupportedDataTypes = tuple(
supportedPinDataTypes)
p.supportedDataTypes = supportedDataTypes
if constraint is not None:
p.updateConstraint(constraint)
if structConstraint is not None:
p.updateStructConstraint(structConstraint)
return p
def createOutputPin(self,
pinName,
dataType,
defaultValue=None,
structure=PinStructure.Single,
constraint=None,
structConstraint=None,
supportedPinDataTypes=[],
group=""):
"""Creates output pin
:param pinName: Pin name
:type pinName: str
:param dataType: Pin data type
:type dataType: str
:param defaultValue: Pin default value
:type defaultValue: object
:param structure: Pin structure
:type structure: :class:`~PyFlow.Core.Common.PinStructure.Single`
:param constraint: Pin constraint. Should be any hashable type. We use str
:type constraint: object
:param structConstraint: Pin struct constraint. Also should be hashable type
:type structConstraint: object
:param supportedPinDataTypes: List of allowed pin data types to be connected. Used by AnyPin
:type supportedPinDataTypes: list(str)
:param group: Pin group. Used only by ui wrapper
:type group: str
"""
pinName = self.getUniqPinName(pinName)
p = CreateRawPin(pinName, self, dataType, PinDirection.Output)
p.structureType = structure
p.group = group
if structure == PinStructure.Array:
p.initAsArray(True)
elif structure == PinStructure.Dict:
p.initAsDict(True)
elif structure == PinStructure.Multi:
p.enableOptions(PinOptions.ArraySupported)
if defaultValue is not None or dataType == "AnyPin":
p.setDefaultValue(defaultValue)
p.setData(defaultValue)
if dataType == "AnyPin":
p.setTypeFromData(defaultValue)
else:
p.setDefaultValue(getPinDefaultValueByType(dataType))
if dataType == "AnyPin" and supportedPinDataTypes:
def supportedDataTypes():
return supportedPinDataTypes
p.supportedDataTypes = supportedDataTypes
if constraint is not None:
p.updateConstraint(constraint)
if structConstraint is not None:
p.updateStructConstraint(structConstraint)
return p
def setData(self,
pinName,
data,
pinSelectionGroup=PinSelectionGroup.BothSides):
"""Sets data to pin by pin name
:param pinName: Target pin name
:type pinName: str
:param data: Pin data to be set
:type data: object
:param pinSelectionGroup: Which side to search
:type pinSelectionGroup: :class:`~PyFlow.Core.Common.PinSelectionGroup`
"""
p = self.getPinSG(str(pinName), pinSelectionGroup)
assert (p
is not None), "Failed to find pin by name: {}".format(pinName)
p.setData(data)
def getData(self, pinName, pinSelectionGroup=PinSelectionGroup.BothSides):
"""Get data from pin by name
:param pinName: Target pin name
:type pinName: str
:param pinSelectionGroup: Which side to search
:type pinSelectionGroup: :class:`~PyFlow.Core.Common.PinSelectionGroup`
:rtype: object
"""
p = self.getPinSG(str(pinName), pinSelectionGroup)
assert (p
is not None), "Failed to find pin by name: {}".format(pinName)
return p.getData()
def getUniqPinName(self, name):
"""Returns unique name for pin
:param name: Target pin name
:type name: str
:rtype: str
"""
pinNames = [
i.name for i in list(list(self.inputs.values())) +
list(list(self.outputs.values()))
]
return getUniqNameFromList(pinNames, name)
def __repr__(self):
graphName = self.graph().name if self.graph is not None else str(None)
return "<class[{0}]; name[{1}]; graph[{2}]>".format(
self.__class__.__name__, self.getName(), graphName)
def call(self, name, *args, **kwargs):
"""Call exec pin by name
:param name: Target pin name
:type name: str
"""
namePinOutputsMap = self.namePinOutputsMap
namePinInputsMap = self.namePinInputsMap
if name in namePinOutputsMap:
p = namePinOutputsMap[name]
if p.isExec():
p.call(*args, **kwargs)
if name in namePinInputsMap:
p = namePinInputsMap[name]
if p.isExec():
p.call(*args, **kwargs)
def getPinSG(self, name, pinsSelectionGroup=PinSelectionGroup.BothSides):
"""Tries to find pin by name and selection group
:param name: Pin name to search
:type name: str
:param pinsSelectionGroup: Side to search
:type pinsSelectionGroup: :class:`~PyFlow.Core.Common.PinSelectionGroup`
:rtype: :class:`~PyFlow.Core.PinBase.PinBase` or None
"""
inputs = self.inputs
outputs = self.outputs
if pinsSelectionGroup == PinSelectionGroup.BothSides:
for p in list(inputs.values()) + list(outputs.values()):
if p.name == name:
return p
elif pinsSelectionGroup == PinSelectionGroup.Inputs:
for p in list(inputs.values()):
if p.name == name:
return p
else:
for p in list(outputs.values()):
if p.name == name:
return p
def getPinByName(self, name):
"""Tries to find pin by name
:param name: pin name
:type name: str
:rtype: :class:`~PyFlow.Core.PinBase.PinBase` or None
"""
inputs = self.inputs
outputs = self.outputs
for p in list(inputs.values()) + list(outputs.values()):
if p.name == name:
return p
if uid in inputs:
return inputs[uid]
if uid in outputs:
return outputs[uid]
return None
def postCreate(self, jsonTemplate=None):
"""Called after node was added to graph
:param jsonTemplate: Serialized data of spawned node
:type jsonTemplate: dict or None
"""
if jsonTemplate is not None:
self.uid = uuid.UUID(jsonTemplate['uuid'])
self.setName(jsonTemplate['name'])
self.x = jsonTemplate['x']
self.y = jsonTemplate['y']
# set pins data
sortedInputs = sorted(jsonTemplate['inputs'],
key=lambda pinDict: pinDict["pinIndex"])
for inpJson in sortedInputs:
dynamicEnabled = PinOptions.Dynamic.value in inpJson["options"]
if dynamicEnabled or inpJson[
'name'] not in self.namePinInputsMap:
# create custom dynamically created pins in derived classes
continue
pin = self.getPinSG(str(inpJson['name']),
PinSelectionGroup.Inputs)
pin.deserialize(inpJson)
sortedOutputs = sorted(jsonTemplate['outputs'],
key=lambda pinDict: pinDict["pinIndex"])
for outJson in sortedOutputs:
dynamicEnabled = PinOptions.Dynamic.value in outJson["options"]
if dynamicEnabled or outJson[
'name'] not in self.namePinOutputsMap:
# create custom dynamically created pins in derived classes
continue
pin = self.getPinSG(str(outJson['name']),
PinSelectionGroup.Outputs)
pin.deserialize(outJson)
# store data for wrapper
if "wrapper" in jsonTemplate:
self.__wrapperJsonData = jsonTemplate["wrapper"]
if self.isCallable():
self.bCallable = True
# make no sense cache nodes without inputs
if len(self.inputs) == 0:
self.bCacheEnabled = False
self.autoAffectPins()
self.checkForErrors()
@staticmethod
def initializeFromFunction(foo):
"""Constructs node from annotated function
.. seealso :: :mod:`PyFlow.Core.FunctionLibrary`
:param foo: Annotated function
:type foo: function
:rtype: :class:`~PyFlow.Core.NodeBase.NodeBase`
"""
retAnyOpts = None
retConstraint = None
meta = foo.__annotations__['meta']
returnType = returnDefaultValue = None
returnPinOptionsToEnable = None
returnPinOptionsToDisable = None
returnWidgetVariant = "DefaultWidget"
retStructConstraint = None
returnAnnotationDict = None
if foo.__annotations__['return'] is not None:
returnType = foo.__annotations__['return'][0]
returnDefaultValue = foo.__annotations__['return'][1]
if len(foo.__annotations__['return']) == 3:
returnAnnotationDict = foo.__annotations__['return'][2]
if "supportedDataTypes" in returnAnnotationDict:
retAnyOpts = returnAnnotationDict["supportedDataTypes"]
if "constraint" in returnAnnotationDict:
retConstraint = returnAnnotationDict["constraint"]
if "structConstraint" in returnAnnotationDict:
retStructConstraint = returnAnnotationDict[
"structConstraint"]
if "enabledOptions" in returnAnnotationDict:
returnPinOptionsToEnable = returnAnnotationDict[
"enabledOptions"]
if "disabledOptions" in returnAnnotationDict:
returnPinOptionsToDisable = returnAnnotationDict[
"disabledOptions"]
if "inputWidgetVariant" in returnAnnotationDict:
returnWidgetVariant = returnAnnotationDict[
"inputWidgetVariant"]
nodeType = foo.__annotations__['nodeType']
_packageName = foo.__annotations__['packageName']
libName = foo.__annotations__['lib']
fooArgNames = getargspec(foo).args
@staticmethod
def description():
return foo.__doc__
@staticmethod
def category():
return meta['Category']
@staticmethod
def keywords():
return meta['Keywords']
def constructor(self, name, **kwargs):
NodeBase.__init__(self, name, **kwargs)
nodeClass = type(
foo.__name__, (NodeBase, ), {
'__init__': constructor,
'category': category,
'keywords': keywords,
'description': description
})
nodeClass._packageName = _packageName
raw_inst = nodeClass(foo.__name__)
raw_inst.lib = libName
# this is list of 'references' outputs will be created for
refs = []
outExec = None
# generate compute method from function
def compute(self, *args, **kwargs):
# arguments will be taken from inputs
if not self.isValid():
return
kwds = {}
for i in list(self.inputs.values()):
if not i.isExec():
kwds[i.name] = i.getData()
for ref in refs:
if not ref.isExec():
kwds[ref.name] = ref.setData
result = foo(**kwds)
if returnType is not None:
self.setData(str('out'), result)
if nodeType == NodeTypes.Callable:
outExec.call(*args, **kwargs)
raw_inst.compute = MethodType(compute, raw_inst)
raw_inst._nodeMetaData = meta
if 'CacheEnabled' in meta:
raw_inst.bCacheEnabled = meta['CacheEnabled']
# create execs if callable
if nodeType == NodeTypes.Callable:
inputExec = raw_inst.createInputPin(DEFAULT_IN_EXEC_NAME,
'ExecPin', None,
raw_inst.compute)
outExec = raw_inst.createOutputPin(DEFAULT_OUT_EXEC_NAME,
'ExecPin')
raw_inst.bCallable = True
raw_inst.bCacheEnabled = False
if returnType is not None:
p = raw_inst.createOutputPin('out',
returnType,
returnDefaultValue,
supportedPinDataTypes=retAnyOpts,
constraint=retConstraint,
structConstraint=retStructConstraint)
p.setData(returnDefaultValue)
p.setDefaultValue(returnDefaultValue)
p.initAsArray(isinstance(returnDefaultValue, list))
p.setInputWidgetVariant(returnWidgetVariant)
p.annotationDescriptionDict = copy(
returnAnnotationDict
) if returnAnnotationDict is not None else None
if p.annotationDescriptionDict is not None and "Description" in p.annotationDescriptionDict:
p.description = p.annotationDescriptionDict["Description"]
if returnPinOptionsToEnable is not None:
p.enableOptions(returnPinOptionsToEnable)
if returnPinOptionsToDisable is not None:
p.disableOptions(returnPinOptionsToDisable)
if not p.isArray() and p.optionEnabled(PinOptions.ArraySupported):
p.structureType = PinStructure.Multi
elif p.isArray():
p.structureType = PinStructure.Array
# iterate over function arguments and create pins according to data types
for index in range(len(fooArgNames)):
argName = fooArgNames[index]
pinDescriptionTuple = foo.__annotations__[argName]
anyOpts = None
constraint = None
structConstraint = None
pinOptionsToEnable = None
pinOptionsToDisable = None
inputWidgetVariant = "DefaultWidget"
# tuple means this is reference pin with default value eg - (dataType, defaultValue)
if str("Reference") == pinDescriptionTuple[0]:
pinDataType = pinDescriptionTuple[1][0]
pinDefaultValue = pinDescriptionTuple[1][1]
pinDict = None
if len(pinDescriptionTuple[1]) == 3:
pinDict = pinDescriptionTuple[1][2]
if pinDict is not None:
if "supportedDataTypes" in pinDict:
anyOpts = pinDict["supportedDataTypes"]
if "constraint" in pinDict:
constraint = pinDict["constraint"]
if "structConstraint" in pinDict:
structConstraint = pinDict["structConstraint"]
if "enabledOptions" in pinDict:
pinOptionsToEnable = pinDict["enabledOptions"]
if "disabledOptions" in pinDict:
pinOptionsToDisable = pinDict["disabledOptions"]
if "inputWidgetVariant" in pinDict:
inputWidgetVariant = pinDict["inputWidgetVariant"]
outRef = raw_inst.createOutputPin(
argName,
pinDataType,
supportedPinDataTypes=anyOpts,
constraint=constraint,
structConstraint=structConstraint)
outRef.annotationDescriptionDict = copy(
pinDict) if pinDict is not None else None
if outRef.annotationDescriptionDict is not None and "Description" in outRef.annotationDescriptionDict:
outRef.description = outRef.annotationDescriptionDict[
"Description"]
outRef.initAsArray(isinstance(pinDefaultValue, list))
outRef.setDefaultValue(pinDefaultValue)
outRef.setData(pinDefaultValue)
outRef.setInputWidgetVariant(inputWidgetVariant)
if pinOptionsToEnable is not None:
outRef.enableOptions(pinOptionsToEnable)
if pinOptionsToDisable is not None:
outRef.disableOptions(pinOptionsToDisable)
if not outRef.isArray() and outRef.optionEnabled(
PinOptions.ArraySupported):
outRef.structureType = PinStructure.Multi
elif outRef.isArray():
outRef.structureType = PinStructure.Array
refs.append(outRef)
else:
pinDataType = pinDescriptionTuple[0]
pinDefaultValue = pinDescriptionTuple[1]
pinDict = None
if len(pinDescriptionTuple) == 3:
pinDict = pinDescriptionTuple[2]
if pinDict is not None:
if "supportedDataTypes" in pinDict:
anyOpts = pinDict["supportedDataTypes"]
if "constraint" in pinDict:
constraint = pinDict["constraint"]
if "structConstraint" in pinDict:
structConstraint = pinDict["structConstraint"]
if "enabledOptions" in pinDict:
pinOptionsToEnable = pinDict["enabledOptions"]
if "disabledOptions" in pinDict:
pinOptionsToDisable = pinDict["disabledOptions"]
if "inputWidgetVariant" in pinDict:
inputWidgetVariant = pinDict["inputWidgetVariant"]
inp = raw_inst.createInputPin(
argName,
pinDataType,
supportedPinDataTypes=anyOpts,
constraint=constraint,
structConstraint=structConstraint)
inp.annotationDescriptionDict = copy(
pinDict) if pinDict is not None else None
if inp.annotationDescriptionDict is not None and "Description" in inp.annotationDescriptionDict:
inp.description = inp.annotationDescriptionDict[
"Description"]
inp.initAsArray(isinstance(pinDefaultValue, list))
inp.setData(pinDefaultValue)
inp.setDefaultValue(pinDefaultValue)
inp.setInputWidgetVariant(inputWidgetVariant)
if pinOptionsToEnable is not None:
inp.enableOptions(pinOptionsToEnable)
if pinOptionsToDisable is not None:
inp.disableOptions(pinOptionsToDisable)
if not inp.isArray() and inp.optionEnabled(
PinOptions.ArraySupported):
inp.structureType = PinStructure.Multi
elif inp.isArray():
inp.structureType = PinStructure.Array
raw_inst.autoAffectPins()
return raw_inst
class UploadedFileManager(CacheStatsProvider):
"""Holds files uploaded by users of the running Streamlit app,
and emits an event signal when a file is added.
"""
def __init__(self):
# List of files for a given widget in a given session.
self._files_by_id: Dict[Tuple[str, str], List[UploadedFileRec]] = {}
# A counter that generates unique file IDs. Each file ID is greater
# than the previous ID, which means we can use IDs to compare files
# by age.
self._file_id_counter = 1
self._file_id_lock = threading.Lock()
# Prevents concurrent access to the _files_by_id dict.
# In remove_session_files(), we iterate over the dict's keys. It's
# an error to mutate a dict while iterating; this lock prevents that.
self._files_lock = threading.Lock()
self.on_files_updated = Signal(
doc="""Emitted when a file list is added to the manager or updated.
Parameters
----------
session_id : str
The session_id for the session whose files were updated.
""")
def __repr__(self) -> str:
return util.repr_(self)
def add_file(
self,
session_id: str,
widget_id: str,
file: UploadedFileRec,
) -> UploadedFileRec:
"""Add a file to the FileManager, and return a new UploadedFileRec
with its ID assigned.
The "on_files_updated" Signal will be emitted.
Parameters
----------
session_id
The session ID of the report that owns the files.
widget_id
The widget ID of the FileUploader that created the files.
file
The file to add.
Returns
-------
UploadedFileRec
The added file, which has its unique ID assigned.
"""
files_by_widget = session_id, widget_id
# Assign the file a unique ID
file_id = self._get_next_file_id()
file = UploadedFileRec(id=file_id,
name=file.name,
type=file.type,
data=file.data)
with self._files_lock:
file_list = self._files_by_id.get(files_by_widget, None)
if file_list is not None:
file_list.append(file)
else:
self._files_by_id[files_by_widget] = [file]
self.on_files_updated.send(session_id)
return file
def get_all_files(self, session_id: str,
widget_id: str) -> List[UploadedFileRec]:
"""Return all the files stored for the given widget.
Parameters
----------
session_id
The session ID of the report that owns the file.
widget_id
The widget ID of the FileUploader that created the file.
"""
file_list_id = (session_id, widget_id)
with self._files_lock:
return self._files_by_id.get(file_list_id, []).copy()
def get_files(self, session_id: str, widget_id: str,
file_ids: List[int]) -> List[UploadedFileRec]:
"""Return the files with the given widget_id and file_ids.
Parameters
----------
session_id
The session ID of the report that owns the file.
widget_id
The widget ID of the FileUploader that created the file.
file_ids
List of file IDs. Only files whose IDs are in this list will be
returned.
"""
return [
f for f in self.get_all_files(session_id, widget_id)
if f.id in file_ids
]
def remove_orphaned_files(
self,
session_id: str,
widget_id: str,
newest_file_id: int,
active_file_ids: List[int],
) -> None:
"""Remove 'orphaned' files: files that have been uploaded and
subsequently deleted, but haven't yet been removed from memory.
Because FileUploader can live inside forms, file deletion is made a
bit tricky: a file deletion should only happen after the form is
submitted.
FileUploader's widget value is an array of numbers that has two parts:
- The first number is always 'this.state.newestServerFileId'.
- The remaining 0 or more numbers are the file IDs of all the
uploader's uploaded files.
When the server receives the widget value, it deletes "orphaned"
uploaded files. An orphaned file is any file associated with a given
FileUploader whose file ID is not in the active_file_ids, and whose
ID is <= `newestServerFileId`.
This logic ensures that a FileUploader within a form doesn't have any
of its "unsubmitted" uploads prematurely deleted when the script is
re-run.
"""
file_list_id = (session_id, widget_id)
with self._files_lock:
file_list = self._files_by_id.get(file_list_id)
if file_list is None:
return
# Remove orphaned files from the list:
# - `f.id in active_file_ids`:
# File is currently tracked by the widget. DON'T remove.
# - `f.id > newest_file_id`:
# file was uploaded *after* the widget was most recently
# updated. (It's probably in a form.) DON'T remove.
# - `f.id < newest_file_id and f.id not in active_file_ids`:
# File is not currently tracked by the widget, and was uploaded
# *before* this most recent update. This means it's been deleted
# by the user on the frontend, and is now "orphaned". Remove!
new_list = [
f for f in file_list
if f.id > newest_file_id or f.id in active_file_ids
]
self._files_by_id[file_list_id] = new_list
num_removed = len(file_list) - len(new_list)
if num_removed > 0:
LOGGER.debug("Removed %s orphaned files" % num_removed)
def remove_file(self, session_id: str, widget_id: str,
file_id: int) -> bool:
"""Remove the file list with the given ID, if it exists.
The "on_files_updated" Signal will be emitted.
Returns
-------
bool
True if the file was removed, or False if no such file exists.
"""
file_list_id = (session_id, widget_id)
with self._files_lock:
file_list = self._files_by_id.get(file_list_id, None)
if file_list is None:
return False
# Remove the file from its list.
new_file_list = [file for file in file_list if file.id != file_id]
self._files_by_id[file_list_id] = new_file_list
self.on_files_updated.send(session_id)
return True
def _remove_files(self, session_id: str, widget_id: str) -> None:
"""Remove the file list for the provided widget in the
provided session, if it exists.
Does not emit any signals.
"""
files_by_widget = session_id, widget_id
with self._files_lock:
self._files_by_id.pop(files_by_widget, None)
def remove_files(self, session_id: str, widget_id: str) -> None:
"""Remove the file list for the provided widget in the
provided session, if it exists.
The "on_files_updated" Signal will be emitted.
Parameters
----------
session_id : str
The session ID of the report that owns the file.
widget_id : str
The widget ID of the FileUploader that created the file.
"""
self._remove_files(session_id, widget_id)
self.on_files_updated.send(session_id)
def remove_session_files(self, session_id: str) -> None:
"""Remove all files that belong to the given report.
Parameters
----------
session_id : str
The session ID of the report whose files we're removing.
"""
# Copy the keys into a list, because we'll be mutating the dictionary.
with self._files_lock:
all_ids = list(self._files_by_id.keys())
for files_id in all_ids:
if files_id[0] == session_id:
self.remove_files(*files_id)
def _get_next_file_id(self) -> int:
"""Return the next file ID and increment our ID counter."""
with self._file_id_lock:
file_id = self._file_id_counter
self._file_id_counter += 1
return file_id
def get_stats(self) -> List[CacheStat]:
with self._files_lock:
# Flatten all files into a single list
all_files: List[UploadedFileRec] = []
for file_list in self._files_by_id.values():
all_files.extend(file_list)
return [
CacheStat(
category_name="UploadedFileManager",
cache_name="",
byte_length=len(file.data),
) for file in all_files
]
def _send_signal(self, sig: Signal, **data: Any) -> None:
sig.send(self.kuyruk, task=self, **data)
class BaseProcess(object):
def __init__(self, new_func=None, total_count=None, *args, **kwargs):
self.func = new_func
self.total_count = total_count
self.count = 0
self.task_queue = []
self.process_list = []
self.process_map = {}
self.task_map = {}
self.args = args
self.kwargs = kwargs
signal.signal(signal.SIGWINCH, self.action)
# self.window = curses.initscr()
self.add_task_signal = Signal(1000)
self.count_signal = Signal(1001)
self.reg_task_signal()
self.reg_count_sinal()
def reg_count_sinal(self):
if not self.total_count:
return
else:
self.count_signal.connect(self.process_count)
def reg_task_signal(self):
self.add_task_signal.connect(self.process_func)
def create_new_task(self, work_num=1):
for i in range(work_num):
self.task_queue.append((self.func, self.args, self.kwargs))
self.add_task_signal.send()
if self.total_count:
for i in range(self.total_count):
self.count_signal.send()
self.process_map = {x: 0 for x in range(work_num)}
def make_func(self):
def add_count_deco(self, func):
def wrapper(func):
@functools.wraps(func)
def _wrapper(*args, **kwargs):
func(*args, **kwargs)
self.count += 1
return _wrapper
return wrapper
@add_count_deco(self,self.func)
def _make_func(*args,**kwargs):
pass
return _make_func
def process_func(self, sender):
while self.task_queue != []:
self.task_queue.pop()
process = multiprocessing.Process(target=self.func, args=self.args, kwargs=self.kwargs)
process.start()
def process_count(self, sender):
self.count += 1
bar_length = 20
percent = float(self.count * 1.0 / self.total_count)
hashes = '#' * int(percent * bar_length)
spaces = ' ' * (bar_length - len(hashes))
multiprocessing.Process(
target=sys.stdout.write("\rPercent: [%s] %d%%" % (hashes + spaces, percent * 100))).start()
def action(self):
key = self.window.getch()
if key == ord('p'):
pass
if key == ord('r'):
pass
def start(self):
while True:
self.action()
class NodeBase(INode):
_packageName = ""
def __init__(self, name, uid=None):
super(NodeBase, self).__init__()
# memo
self.bCacheEnabled = True
self.cacheMaxSize = 1000
self.cache = {}
self.killed = Signal()
self.tick = Signal(float)
self.errorOccured = Signal(object)
self.errorCleared = Signal()
self._uid = uuid.uuid4() if uid is None else uid
self.graph = None
self.name = name
self.pinsCreationOrder = OrderedDict()
self._pins = set()
self.x = 0.0
self.y = 0.0
self.bCallable = False
self._wrapper = None
self._constraints = {}
self._structConstraints = {}
self.lib = None
self.isCompoundNode = False
self._lastError = None
self.__wrapperJsonData = None
@property
def wrapperJsonData(self):
try:
dt = self.__wrapperJsonData.copy()
self.__wrapperJsonData.clear()
self.__wrapperJsonData = None
return dt
except:
return None
def isValid(self):
return self._lastError is None
def clearError(self):
self._lastError = None
self.errorCleared.send()
def setError(self, err):
self._lastError = str(err)
self.errorOccured.send(self._lastError)
@property
def packageName(self):
return self._packageName
@property
def constraints(self):
return self._constraints
@property
def structConstraints(self):
return self._structConstraints
def getOrderedPins(self):
return self.pinsCreationOrder.values()
@dispatch(str)
def __getitem__(self, pinName):
return self.getPin(pinName)
@property
def pins(self):
return self._pins
@property
def inputs(self):
"""Returns all input pins. Dictionary generated every time property called, so cache it when possible
Returns:
dict(uuid: PinBase)
"""
result = OrderedDict()
for pin in self.pins:
if pin.direction == PinDirection.Input:
result[pin.uid] = pin
return result
@property
def namePinInputsMap(self):
"""Returns all input pins. Dictionary generated every time property called, so cache it when possible
Returns:
dict(str: PinBase)
"""
result = OrderedDict()
for pin in self.pins:
if pin.direction == PinDirection.Input:
result[pin.name] = pin
return result
@property
def outputs(self):
"""Returns all output pins. Dictionary generated every time property called, so cache it when possible
Returns:
dict(uuid: PinBase)
"""
result = OrderedDict()
for pin in self.pins:
if pin.direction == PinDirection.Output:
result[pin.uid] = pin
return result
@property
def namePinOutputsMap(self):
"""Returns all output pins. Dictionary generated every time property called, so cache it when possible
Returns:
dict(str: PinBase)
"""
result = OrderedDict()
for pin in self.pins:
if pin.direction == PinDirection.Output:
result[pin.name] = pin
return result
# IItemBase interface
def setWrapper(self, wrapper):
if self._wrapper is None:
self._wrapper = wrapper
def getWrapper(self):
return self._wrapper
@property
def uid(self):
return self._uid
@uid.setter
def uid(self, value):
if self.graph is not None:
self.graph().nodes[value] = self.graph().nodes.pop(self._uid)
self._uid = value
@staticmethod
def jsonTemplate():
template = {
'package': None,
'lib': None,
'type': None,
'owningGraphName': None,
'name': None,
'uuid': None,
'inputs': [],
'outputs': [],
'meta': {
'var': {}
},
'wrapper': {}
}
return template
def serialize(self):
template = NodeBase.jsonTemplate()
uidString = str(self.uid)
nodeName = self.name
template['package'] = self.packageName
template['lib'] = self.lib
template['type'] = self.__class__.__name__
template['name'] = nodeName
template['owningGraphName'] = self.graph().name
template['uuid'] = uidString
template['inputs'] = [i.serialize() for i in self.inputs.values()]
template['outputs'] = [o.serialize() for o in self.outputs.values()]
template['meta']['label'] = self.name
template['x'] = self.x
template['y'] = self.y
# if running with ui get ui wrapper data to save
wrapper = self.getWrapper()
if wrapper:
template['wrapper'] = wrapper.serializationHook()
return template
def isUnderActiveGraph(self):
return self.graph() == self.graph().graphManager.activeGraph()
def kill(self, *args, **kwargs):
if self.uid not in self.graph().nodes:
# already killed
# this block executes for variable getter/setter
return
self.killed.send()
for pin in self.inputs.values():
pin.kill()
for pin in self.outputs.values():
pin.kill()
self.graph().nodes.pop(self.uid)
def Tick(self, delta):
self.tick.send(delta)
@staticmethod
def category():
return "Default"
@staticmethod
def keywords():
return []
@staticmethod
def pinTypeHints():
return {'inputs': [], 'outputs': []}
@staticmethod
def description():
return "Default node description"
def getName(self):
return self.name
def setName(self, name):
# self.graph is sometimes still none
try:
assert self.graph != None
self.name = self.graph().graphManager.getUniqNodeName(str(name))
except:
self.name = name
def useCache(self):
# if cached results exists - return them without calling compute
args = tuple([pin.currentData() for pin in self.inputs.values()])
try:
# mutable unhashable types will not be cached
if args in self.cache:
for outPin, data in self.cache[args].items():
outPin.setData(data)
return True
except:
return False
def afterCompute(self):
if len(self.cache) >= self.cacheMaxSize:
return
# cache results
args = tuple([pin.currentData() for pin in self.inputs.values()])
try:
# mutable unhashable types will not be cached
if args in self.cache:
return
except:
return
cache = {}
for pin in self.outputs.values():
cache[pin] = pin.currentData()
self.cache[args] = cache
def processNode(self, *args, **kwargs):
if not self.isValid():
return
if self.bCacheEnabled:
if not self.useCache():
try:
self.compute()
self.clearError()
except Exception as e:
self.setError(e)
self.afterCompute()
else:
self.compute()
# INode interface
def compute(self, *args, **kwargs):
'''
node calculations here
'''
# getting data from inputs
# do stuff
# write data to outputs
return
# INode interface end
def isCallable(self):
for p in list(self.inputs.values()) + list(self.outputs.values()):
if p.isExec():
return True
return False
def setPosition(self, x, y):
self.x = x
self.y = y
def autoAffectPins(self):
"""All value inputs affects on all value outputs. All exec inputs affects on all exec outputs
"""
for i in self.inputs.values():
for o in self.outputs.values():
assert (i is not o)
if not i.IsValuePin() and o.IsValuePin():
continue
if i.IsValuePin() and not o.IsValuePin():
continue
pinAffects(i, o)
def createInputPin(self,
pinName,
dataType,
defaultValue=None,
foo=None,
structure=PinStructure.Single,
constraint=None,
structConstraint=None,
allowedPins=[],
group=""):
# check unique name
pinName = self.getUniqPinName(pinName)
p = CreateRawPin(pinName, self, dataType, PinDirection.Input)
p.structureType = structure
p.group = group
if structure == PinStructure.Array:
p.initAsArray(True)
elif structure == PinStructure.Multi:
p.enableOptions(PinOptions.ArraySupported)
if foo:
p.onExecute.connect(foo, weak=False)
if defaultValue is not None:
p.setDefaultValue(defaultValue)
p.setData(defaultValue)
else:
p.setDefaultValue(getPinDefaultValueByType(dataType))
if dataType == "AnyPin" and allowedPins:
def supportedDataTypes():
return allowedPins
p._supportedDataTypes = p._defaultSupportedDataTypes = tuple(
allowedPins)
p.supportedDataTypes = supportedDataTypes
if constraint is not None:
p.updateConstraint(constraint)
if structConstraint is not None:
p.updatestructConstraint(structConstraint)
return p
def createOutputPin(self,
pinName,
dataType,
defaultValue=None,
structure=PinStructure.Single,
constraint=None,
structConstraint=None,
allowedPins=[],
group=""):
pinName = self.getUniqPinName(pinName)
p = CreateRawPin(pinName, self, dataType, PinDirection.Output)
p.structureType = structure
p.group = group
if structure == PinStructure.Array:
p.initAsArray(True)
elif structure == PinStructure.Multi:
p.enableOptions(PinOptions.ArraySupported)
if defaultValue is not None:
p.setDefaultValue(defaultValue)
p.setData(defaultValue)
else:
p.setDefaultValue(getPinDefaultValueByType(dataType))
if dataType == "AnyPin" and allowedPins:
def supportedDataTypes():
return allowedPins
p.supportedDataTypes = supportedDataTypes
if constraint is not None:
p.updateConstraint(constraint)
if structConstraint is not None:
p.updatestructConstraint(structConstraint)
return p
def setData(self,
pinName,
data,
pinSelectionGroup=PinSelectionGroup.BothSides):
p = self.getPin(str(pinName), pinSelectionGroup)
assert (p
is not None), "Failed to find pin by name: {}".format(pinName)
p.setData(data)
def getData(self, pinName, pinSelectionGroup=PinSelectionGroup.BothSides):
p = self.getPin(str(pinName), pinSelectionGroup)
assert (p
is not None), "Failed to find pin by name: {}".format(pinName)
return p.currentData()
def getUniqPinName(self, name):
pinNames = [
i.name for i in list(list(self.inputs.values())) +
list(list(self.outputs.values()))
]
return getUniqNameFromList(pinNames, name)
def __repr__(self):
graphName = self.graph().name if self.graph is not None else str(None)
return "<class[{0}]; name[{1}]; graph[{2}]>".format(
self.__class__.__name__, self.getName(), graphName)
def call(self, name, *args, **kwargs):
namePinOutputsMap = self.namePinOutputsMap
namePinInputsMap = self.namePinInputsMap
if name in namePinOutputsMap:
p = namePinOutputsMap[name]
if p.isExec():
p.call(*args, **kwargs)
if name in namePinInputsMap:
p = namePinInputsMap[name]
if p.isExec():
p.call(*args, **kwargs)
@dispatch(str, PinSelectionGroup)
def getPin(self, name, pinsSelectionGroup=PinSelectionGroup.BothSides):
inputs = self.inputs
outputs = self.outputs
if pinsSelectionGroup == PinSelectionGroup.BothSides:
for p in list(inputs.values()) + list(outputs.values()):
if p.name == name:
return p
elif pinsSelectionGroup == PinSelectionGroup.Inputs:
for p in list(inputs.values()):
if p.name == name:
return p
else:
for p in list(outputs.values()):
if p.name == name:
return p
@dispatch(str)
def getPin(self, name):
inputs = self.inputs
outputs = self.outputs
for p in list(inputs.values()) + list(outputs.values()):
if p.name == name:
return p
@dispatch(uuid.UUID)
def getPin(self, uid):
inputs = self.inputs
outputs = self.outputs
if uid in inputs:
return inputs[uid]
if uid in outputs:
return outputs[uid]
return None
def postCreate(self, jsonTemplate=None):
if jsonTemplate is not None:
self.uid = uuid.UUID(jsonTemplate['uuid'])
self.setName(jsonTemplate['name'])
self.x = jsonTemplate['x']
self.y = jsonTemplate['y']
# set pins data
for inpJson in jsonTemplate['inputs']:
dynamicEnabled = PinOptions.Dynamic.value in inpJson["options"]
if dynamicEnabled or inpJson[
'name'] not in self.namePinInputsMap:
# create custom dynamically created pins in derived classes
continue
pin = self.getPin(str(inpJson['name']),
PinSelectionGroup.Inputs)
pin.deserialize(inpJson)
for outJson in jsonTemplate['outputs']:
dynamicEnabled = PinOptions.Dynamic.value in outJson["options"]
if dynamicEnabled or outJson[
'name'] not in self.namePinOutputsMap:
# create custom dynamically created pins in derived classes
continue
pin = self.getPin(str(outJson['name']),
PinSelectionGroup.Outputs)
pin.deserialize(outJson)
# store data for wrapper
if "wrapper" in jsonTemplate:
self.__wrapperJsonData = jsonTemplate["wrapper"]
if self.isCallable():
self.bCallable = True
self.autoAffectPins()
@staticmethod
# Constructs a node from given annotated function
def initializeFromFunction(foo):
retAnyOpts = None
retConstraint = None
foo = foo
meta = foo.__annotations__['meta']
returnType = returnDefaultValue = None
returnPinOptionsToEnable = None
returnPinOptionsToDisable = None
retStructConstraint = None
if foo.__annotations__['return'] is not None:
returnType = foo.__annotations__['return'][0]
returnDefaultValue = foo.__annotations__['return'][1]
if len(foo.__annotations__['return']) == 3:
if "supportedDataTypes" in foo.__annotations__['return'][2]:
retAnyOpts = foo.__annotations__['return'][2][
"supportedDataTypes"]
if "constraint" in foo.__annotations__['return'][2]:
retConstraint = foo.__annotations__['return'][2][
"constraint"]
if "structConstraint" in foo.__annotations__['return'][2]:
retStructConstraint = foo.__annotations__['return'][2][
"structConstraint"]
if "enabledOptions" in foo.__annotations__['return'][2]:
returnPinOptionsToEnable = foo.__annotations__['return'][
2]["enabledOptions"]
if "disabledOptions" in foo.__annotations__['return'][2]:
returnPinOptionsToDisable = foo.__annotations__['return'][
2]["disabledOptions"]
nodeType = foo.__annotations__['nodeType']
_packageName = foo.__annotations__['packageName']
libName = foo.__annotations__['lib']
fooArgNames = getargspec(foo).args
@staticmethod
def description():
return foo.__doc__
@staticmethod
def category():
return meta['Category']
@staticmethod
def keywords():
return meta['Keywords']
def constructor(self, name, **kwargs):
NodeBase.__init__(self, name, **kwargs)
nodeClass = type(
foo.__name__, (NodeBase, ), {
'__init__': constructor,
'category': category,
'keywords': keywords,
'description': description
})
nodeClass._packageName = _packageName
raw_inst = nodeClass(foo.__name__)
raw_inst.lib = libName
# this is list of 'references' outputs will be created for
refs = []
outExec = None
# generate compute method from function
def compute(self, *args, **kwargs):
# arguments will be taken from inputs
kwds = {}
for i in list(self.inputs.values()):
if not i.isExec():
kwds[i.name] = i.getData()
for ref in refs:
if not ref.isExec():
kwds[ref.name] = ref.setData
result = foo(**kwds)
if returnType is not None:
self.setData(str('out'), result)
if nodeType == NodeTypes.Callable:
outExec.call(*args, **kwargs)
raw_inst.compute = MethodType(compute, raw_inst)
if 'CacheEnabled' in meta:
raw_inst.bCacheEnabled = meta['CacheEnabled']
# create execs if callable
if nodeType == NodeTypes.Callable:
inputExec = raw_inst.createInputPin(DEFAULT_IN_EXEC_NAME,
'ExecPin', None,
raw_inst.compute)
outExec = raw_inst.createOutputPin(DEFAULT_OUT_EXEC_NAME,
'ExecPin')
raw_inst.bCallable = True
raw_inst.bCacheEnabled = False
if returnType is not None:
p = raw_inst.createOutputPin('out',
returnType,
returnDefaultValue,
allowedPins=retAnyOpts,
constraint=retConstraint,
structConstraint=retStructConstraint)
p.setData(returnDefaultValue)
p.setDefaultValue(returnDefaultValue)
p.initAsArray(isinstance(returnDefaultValue, list))
if returnPinOptionsToEnable is not None:
p.enableOptions(returnPinOptionsToEnable)
if returnPinOptionsToDisable is not None:
p.disableOptions(returnPinOptionsToDisable)
if not p.isArray() and p.optionEnabled(PinOptions.ArraySupported):
p.structureType = PinStructure.Multi
# iterate over function arguments and create pins according to data types
for index in range(len(fooArgNames)):
argName = fooArgNames[index]
pinDescriptionTuple = foo.__annotations__[argName]
anyOpts = None
constraint = None
structConstraint = None
pinOptionsToEnable = None
pinOptionsToDisable = None
# tuple means this is reference pin with default value eg - (dataType, defaultValue)
if str("Reference") == pinDescriptionTuple[0]:
pinDataType = pinDescriptionTuple[1][0]
pinDefaultValue = pinDescriptionTuple[1][1]
pinDict = None
if len(pinDescriptionTuple[1]) == 3:
pinDict = pinDescriptionTuple[1][2]
if pinDict is not None:
if "supportedDataTypes" in pinDict:
anyOpts = pinDict["supportedDataTypes"]
if "constraint" in pinDict:
constraint = pinDict["constraint"]
if "structConstraint" in pinDict:
structConstraint = pinDict["structConstraint"]
if "enabledOptions" in pinDict:
pinOptionsToEnable = pinDict["enabledOptions"]
if "disabledOptions" in pinDict:
pinOptionsToDisable = pinDict["disabledOptions"]
outRef = raw_inst.createOutputPin(
argName,
pinDataType,
allowedPins=anyOpts,
constraint=constraint,
structConstraint=structConstraint)
outRef.initAsArray(isinstance(pinDefaultValue, list))
outRef.setDefaultValue(pinDefaultValue)
outRef.setData(pinDefaultValue)
if pinOptionsToEnable is not None:
outRef.enableOptions(pinOptionsToEnable)
if pinOptionsToDisable is not None:
outRef.disableOptions(pinOptionsToDisable)
if not outRef.isArray() and outRef.optionEnabled(
PinOptions.ArraySupported):
outRef.structureType = PinStructure.Multi
refs.append(outRef)
else:
pinDataType = pinDescriptionTuple[0]
pinDefaultValue = pinDescriptionTuple[1]
pinDict = None
if len(pinDescriptionTuple) == 3:
pinDict = pinDescriptionTuple[2]
if pinDict is not None:
if "supportedDataTypes" in pinDict:
anyOpts = pinDict["supportedDataTypes"]
if "constraint" in pinDict:
constraint = pinDict["constraint"]
if "structConstraint" in pinDict:
structConstraint = pinDict["structConstraint"]
if "enabledOptions" in pinDict:
pinOptionsToEnable = pinDict["enabledOptions"]
if "disabledOptions" in pinDict:
pinOptionsToDisable = pinDict["disabledOptions"]
inp = raw_inst.createInputPin(
argName,
pinDataType,
allowedPins=anyOpts,
constraint=constraint,
structConstraint=structConstraint)
inp.initAsArray(isinstance(pinDefaultValue, list))
inp.setData(pinDefaultValue)
inp.setDefaultValue(pinDefaultValue)
if pinOptionsToEnable is not None:
inp.enableOptions(pinOptionsToEnable)
if pinOptionsToDisable is not None:
inp.disableOptions(pinOptionsToDisable)
if not inp.isArray() and inp.optionEnabled(
PinOptions.ArraySupported):
inp.structureType = PinStructure.Multi
raw_inst.autoAffectPins()
return raw_inst
class ScriptRunner(object):
def __init__(self, report, main_dg, sidebar_dg, widget_states,
request_queue):
"""Initialize the ScriptRunner.
(The ScriptRunner won't start executing until start() is called.)
Parameters
----------
report : Report
The ReportSession's report.
main_dg : DeltaGenerator
The ReportSession's main DeltaGenerator.
sidebar_dg : DeltaGenerator
The ReportSession's sidebar DeltaGenerator.
widget_states : streamlit.proto.Widget_pb2.WidgetStates
The ReportSession's current widget states
request_queue : ScriptRequestQueue
The queue that the ReportSession is publishing ScriptRequests to.
ScriptRunner will continue running until the queue is empty,
and then shut down.
"""
self._report = report
self._main_dg = main_dg
self._sidebar_dg = sidebar_dg
self._request_queue = request_queue
self._widgets = Widgets()
self._widgets.set_state(widget_states)
self.on_event = Signal(doc="""Emitted when a ScriptRunnerEvent occurs.
This signal is *not* emitted on the same thread that the
ScriptRunner was created on.
Parameters
----------
event : ScriptRunnerEvent
exception : BaseException | None
Our compile error. Set only for the
SCRIPT_STOPPED_WITH_COMPILE_ERROR event.
widget_states : streamlit.proto.Widget_pb2.WidgetStates | None
The ScriptRunner's final WidgetStates. Set only for the
SHUTDOWN event.
""")
# Set to true when we process a SHUTDOWN request
self._shutdown_requested = False
# Set to true while we're executing. Used by
# maybe_handle_execution_control_request.
self._execing = False
# This is initialized in start()
self._script_thread = None
def start(self):
"""Start a new thread to process the ScriptEventQueue.
This must be called only once.
"""
if self._script_thread is not None:
raise Exception("ScriptRunner was already started")
self._script_thread = ReportThread(
main_dg=self._main_dg,
sidebar_dg=self._sidebar_dg,
widgets=self._widgets,
target=self._process_request_queue,
name="ScriptRunner.scriptThread",
)
self._script_thread.start()
def _process_request_queue(self):
"""Process the ScriptRequestQueue and then exits.
This is run in a separate thread.
"""
LOGGER.debug("Beginning script thread")
while not self._shutdown_requested and self._request_queue.has_request:
request, data = self._request_queue.dequeue()
if request == ScriptRequest.STOP:
LOGGER.debug("Ignoring STOP request while not running")
elif request == ScriptRequest.SHUTDOWN:
LOGGER.debug("Shutting down")
self._shutdown_requested = True
elif request == ScriptRequest.RERUN:
self._run_script(data)
else:
raise RuntimeError("Unrecognized ScriptRequest: %s" % request)
# Send a SHUTDOWN event before exiting. This includes the widget values
# as they existed after our last successful script run, which the
# ReportSession will pass on to the next ScriptRunner that gets
# created.
self.on_event.send(ScriptRunnerEvent.SHUTDOWN,
widget_states=self._widgets.get_state())
def _is_in_script_thread(self):
"""True if the calling function is running in the script thread"""
return self._script_thread == threading.current_thread()
def maybe_handle_execution_control_request(self):
if not self._is_in_script_thread():
# We can only handle execution_control_request if we're on the
# script execution thread. However, it's possible for deltas to
# be enqueued (and, therefore, for this function to be called)
# in separate threads, so we check for that here.
return
if not self._execing:
# If the _execing flag is not set, we're not actually inside
# an exec() call. This happens when our script exec() completes,
# we change our state to STOPPED, and a statechange-listener
# enqueues a new ForwardEvent
return
# Pop the next request from our queue.
request, data = self._request_queue.dequeue()
if request is None:
return
LOGGER.debug("Received ScriptRequest: %s", request)
if request == ScriptRequest.STOP:
raise StopException()
elif request == ScriptRequest.SHUTDOWN:
self._shutdown_requested = True
raise StopException()
elif request == ScriptRequest.RERUN:
raise RerunException(data)
else:
raise RuntimeError("Unrecognized ScriptRequest: %s" % request)
def _install_tracer(self):
"""Install function that runs before each line of the script."""
def trace_calls(frame, event, arg):
self.maybe_handle_execution_control_request()
return trace_calls
# Python interpreters are not required to implement sys.settrace.
if hasattr(sys, "settrace"):
sys.settrace(trace_calls)
@contextmanager
def _set_execing_flag(self):
"""A context for setting the ScriptRunner._execing flag.
Used by maybe_handle_execution_control_request to ensure that
we only handle requests while we're inside an exec() call
"""
if self._execing:
raise RuntimeError("Nested set_execing_flag call")
self._execing = True
try:
yield
finally:
self._execing = False
def _run_script(self, rerun_data):
"""Run our script.
Parameters
----------
rerun_data: RerunData
The RerunData to use.
"""
assert self._is_in_script_thread()
LOGGER.debug("Running script %s", rerun_data)
# Reset delta generator so it starts from index 0.
import streamlit as st
st._reset(self._main_dg, self._sidebar_dg)
self.on_event.send(ScriptRunnerEvent.SCRIPT_STARTED)
# Compile the script. Any errors thrown here will be surfaced
# to the user via a modal dialog in the frontend, and won't result
# in their previous report disappearing.
try:
# Python 3 got rid of the native execfile() command, so we read
# the file, compile it, and exec() it. This implementation is
# compatible with both 2 and 3.
with open(self._report.script_path) as f:
filebody = f.read()
if config.get_option("runner.magicEnabled"):
filebody = magic.add_magic(filebody, self._report.script_path)
code = compile(
filebody,
# Pass in the file path so it can show up in exceptions.
self._report.script_path,
# We're compiling entire blocks of Python, so we need "exec"
# mode (as opposed to "eval" or "single").
mode="exec",
# Don't inherit any flags or "future" statements.
flags=0,
dont_inherit=1,
# Parameter not supported in Python2:
# optimize=-1,
)
except BaseException as e:
# We got a compile error. Send an error event and bail immediately.
LOGGER.debug("Fatal script error: %s" % e)
self.on_event.send(
ScriptRunnerEvent.SCRIPT_STOPPED_WITH_COMPILE_ERROR,
exception=e)
return
# If we get here, we've successfully compiled our script. The next step
# is to run it. Errors thrown during execution will be shown to the
# user as ExceptionElements.
# Update Report.argv
if rerun_data.argv is not None:
argv = rerun_data.argv
self._report.argv = rerun_data.argv
else:
argv = self._report.argv
# Update the Widget singleton with the new widget_state
if rerun_data.widget_state is not None:
self._widgets.set_state(rerun_data.widget_state)
if config.get_option("runner.installTracer"):
self._install_tracer()
# This will be set to a RerunData instance if our execution
# is interrupted by a RerunException.
rerun_with_data = None
try:
# Create fake module. This gives us a name global namespace to
# execute the code in.
module = _new_module("__main__")
# Install the fake module as the __main__ module. This allows
# the pickle module to work inside the user's code, since it now
# can know the module where the pickled objects stem from.
# IMPORTANT: This means we can't use "if __name__ == '__main__'" in
# our code, as it will point to the wrong module!!!
sys.modules["__main__"] = module
# Make it look like command-line args were set to whatever the user
# asked them to be via the GUI.
# IMPORTANT: This means we can't count on sys.argv in our code ---
# but we already knew it from the argv surgery in cli.py.
# TODO: Remove this feature when we implement interactivity!
# This is not robust in a multi-user environment.
sys.argv = argv
# Add special variables to the module's globals dict.
module.__dict__["__file__"] = self._report.script_path
with modified_sys_path(self._report), self._set_execing_flag():
exec(code, module.__dict__)
except RerunException as e:
rerun_with_data = e.rerun_data
except StopException:
pass
except BaseException as e:
# Show exceptions in the Streamlit report.
LOGGER.debug(e)
import streamlit as st
st.exception(e) # This is OK because we're in the script thread.
# TODO: Clean up the stack trace, so it doesn't include
# ScriptRunner.
finally:
self._widgets.reset_triggers()
self.on_event.send(ScriptRunnerEvent.SCRIPT_STOPPED_WITH_SUCCESS)
# Use _log_if_error() to make sure we never ever ever stop running the
# script without meaning to.
_log_if_error(_clean_problem_modules)
if rerun_with_data is not None:
self._run_script(rerun_with_data)
class Settings(abc.Mapping):
"""Specialized mapping for *Value instances"""
def __init__(self):
self._defaults = {}
self._values = {}
self._constructors = {}
self._descriptions = {}
self._signals = {}
self._global_signal = Signal()
def add(self, name, constructor, default, description=None):
"""
Add new setting
name: Identifier for this setting
constructor: Callable that takes one argument and returns a new value
for this setting
default: Initial and default value
description: What the setting does
"""
self._constructors[name] = constructor
self._signals[name] = Signal()
self._descriptions[name] = description
self[name] = default
self._defaults[name] = self[name]
self._global_signal.send(self, name=name, value=self[name])
def reset(self, name):
"""Reset setting `name` to default/initial value"""
self[name] = self._defaults[name]
def default(self, name):
"""Return settings default/initial value"""
return self._defaults[name]
def description(self, name):
"""Return setting's description"""
return self._descriptions[name]
def syntax(self, name):
"""Return setting's description"""
return self._constructors[name].syntax
def validate(self, name, value):
"""Pass `value` to `name`'s constructor and return the result"""
if not isinstance(value, str) and isinstance(value, abc.Iterable):
return self._constructors[name](*value)
else:
return self._constructors[name](value)
def on_change(self, callback, name=None, autoremove=True):
"""
Run `callback` every time a value changes
If `name` is None, run `callback` if any signal changes.
The signature of `callback` must be: (settings, name, value)
If `autoremove` is True, stop calling `callback` once it is garbage
collected.
"""
if name is None:
self._global_signal.connect(callback, weak=autoremove)
else:
self._signals[name].connect(callback, weak=autoremove)
def __getitem__(self, name):
return self._values[name]
def __setitem__(self, name, value):
value_ = self.validate(name, value)
self._values[name] = value_
self._global_signal.send(self, name=name, value=value_)
self._signals[name].send(self, name=name, value=value_)
def __contains__(self, name):
return name in self._constructors
def __iter__(self):
return iter(self._constructors)
def __len__(self):
return len(self._constructors)
class Variable(IItemBase):
def __init__(self,
graph,
value,
name,
dataType,
accessLevel=AccessLevel.public,
structure=PinStructure.Single,
uid=None):
super(Variable, self).__init__()
# signals
self.nameChanged = Signal(str)
self.valueChanged = Signal(str)
self.dataTypeChanged = Signal(str)
self.structureChanged = Signal(str)
self.accessLevelChanged = Signal(str)
self.packageNameChanged = Signal(str)
self.uuidChanged = Signal(object)
self.killed = Signal()
self.graph = graph
self._name = name
self._value = value
self._dataType = dataType
self._structure = structure
self._accessLevel = accessLevel
self._packageName = None
self._uid = uuid.uuid4() if uid is None else uid
assert (isinstance(self._uid, uuid.UUID))
self.updatePackageName()
self._uiWrapper = None
def getWrapper(self):
if self._uiWrapper is not None:
return self._uiWrapper()
return None
def setWrapper(self, wrapper):
if self._uiWrapper is None:
self._uiWrapper = weakref.ref(wrapper)
def location(self):
return self.graph.location()
def findRefs(self):
"""returns all getVar and setVar instances for this node
"""
return self.graph.graphManager.findVariableRefs(self)
def updatePackageName(self):
self._packageName = findPinClassByType(self._dataType)._packageName
@property
def packageName(self):
return self._packageName
@packageName.setter
def packageName(self, value):
assert (isinstance(value, str))
self._packageName = value
self.packageNameChanged.send(value)
@property
def accessLevel(self):
return self._accessLevel
@accessLevel.setter
def accessLevel(self, value):
assert (isinstance(value, AccessLevel))
self._accessLevel = value
self.accessLevelChanged.send(value)
@property
def name(self):
return self._name
@name.setter
def name(self, value):
assert (isinstance(value, str))
self._name = value
self.nameChanged.send(value)
@property
def value(self):
return self._value
@value.setter
def value(self, value):
# type checking if this variable is not of any type
if not self.dataType == 'AnyPin':
supportedDataTypes = findPinClassByType(
self.dataType).supportedDataTypes()
if self.dataType not in supportedDataTypes:
return
try:
if self._value != value:
self._value = value
self.valueChanged.send(value)
except:
self._value = value
self.valueChanged.send(value)
@property
def dataType(self):
return self._dataType
@dataType.setter
def dataType(self, value):
assert (isinstance(value, str))
if value != self._dataType:
self._dataType = value
self.updatePackageName()
self.value = getPinDefaultValueByType(self._dataType)
self.dataTypeChanged.send(value)
@property
def structure(self):
return self._structure
@structure.setter
def structure(self, value):
assert (isinstance(value, PinStructure))
if value != self._structure:
self._structure = value
if self._structure == PinStructure.Array:
self.value = list()
self.structureChanged.send(self._structure)
@property
def uid(self):
return self._uid
@uid.setter
def uid(self, value):
assert (isinstance(value, uuid.UUID))
graph.vars[value] = graph.vars.pop(self._uid)
self._uid = value
def serialize(self):
pinClass = findPinClassByType(self.dataType)
template = Variable.jsonTemplate()
uidString = str(self.uid)
template['name'] = self.name
if self.dataType == 'AnyPin':
template['value'] = None
else:
template['value'] = json.dumps(self.value,
cls=pinClass.jsonEncoderClass())
template['dataType'] = self.dataType
template['structure'] = self.structure.name
template['accessLevel'] = self.accessLevel.name
template['package'] = self._packageName
template['uuid'] = uidString
return template
@staticmethod
def deserialize(graph, jsonData, *args, **kwargs):
name = jsonData['name']
dataType = jsonData['dataType']
if dataType != "AnyPin":
pinClass = findPinClassByType(dataType)
value = json.loads(jsonData['value'],
cls=pinClass.jsonDecoderClass())
else:
value = getPinDefaultValueByType("AnyPin")
accessLevel = AccessLevel[jsonData['accessLevel']]
structure = PinStructure[jsonData['structure']]
uid = uuid.UUID(jsonData['uuid'])
return Variable(graph, value, name, dataType, accessLevel, structure,
uid)
@staticmethod
def jsonTemplate():
template = {
'name': None,
'value': None,
'dataType': None,
'accessLevel': None,
'package': None,
'uuid': None
}
return template
class Axis(object):
def __init__(self, inverted = False, scale={}):
self.inverted = inverted
self.scale = scale
self.position = None
self.running = None
self.initializing = None
self.initialized = None
self.initiator_minus = None
self.initiator_plus = None
self.initiator_error = None
self.temperature_warning = None
self.onPosition = Signal()
self.onStarted = Signal()
self.onStopped = Signal()
self.onInitializing = Signal()
self.onInitialized = Signal()
self.onInitiatorMinus = Signal()
self.onInitiatorPlus = Signal()
self.onInitiatorError = Signal()
self.onTemperatureWarning = Signal()
def update(self):
last_position = self.position
last_running = self.running
last_initializing = self.initializing
last_initialized = self.initialized
last_initiator_minus = self.initiator_minus
last_initiator_plus = self.initiator_plus
last_initiator_error = self.initiator_error
last_temperature_warning = self.temperature_warning
self.do_update()
if last_position != self.position:
self.onPosition.send(position = self.position)
if last_running != self.running:
if self.running:
self.onStarted.send()
else:
self.onStopped.send()
if last_initializing != self.initializing:
self.onInitializing.send(self, initializing = self.initializing)
if last_initialized != self.initialized:
self.onInitialized.send(self, initialized = self.initialized)
if last_initiator_minus != self.initiator_minus:
self.onInitiatorMinus.send(self, active = self.initiator_minus)
if last_initiator_plus != self.initiator_plus:
self.onInitiatorPlus.send(self, active = self.initiator_plus)
if last_initiator_error != self.initiator_error:
self.onInitiatorError.send(self, error = self.initiator_error)
if last_temperature_warning != self.temperature_warning:
self.onTemperatureWarning.send(self, warning = self.temperature_warning)
def wait_for_stop(self):
self.update()
while self.running:
self.update()
def initiate(self):
raise NotImplementedError()
def goto_absolute(self, target, speed = None):
raise NotImplementedError()
def goto_relative(self, offset, speed = None):
raise NotImplementedError()
class compound(NodeBase):
"""This node encapsulates a graph, like compound in xsi
pins can be edited only from inside the compound
"""
def __init__(self, name):
super(compound, self).__init__(name)
self.isCompoundNode = True
self.pinExposed = Signal(object)
self._rawGraph = None
self._rawGraphJson = None
self.__inputsMap = {}
self.__outputsMap = {}
self.bCacheEnabled = True
@property
def inputsMap(self):
return self.__inputsMap
@property
def outputsMap(self):
return self.__outputsMap
@property
def rawGraph(self):
return self._rawGraph
@rawGraph.setter
def rawGraph(self, newGraph):
assert (newGraph is not None)
self._rawGraph = newGraph
def syncPins(self):
# look for graph nodes pins was added
nodeInputPins = self.namePinInputsMap
nodeOutputPins = self.namePinOutputsMap
graphInputsNodes = self.rawGraph.getNodesList(
classNameFilters=['graphInputs'])
graphInputPins = {}
for graphInputNode in graphInputsNodes:
for outPin in graphInputNode.orderedOutputs.values():
graphInputPins[outPin.name] = outPin
# create companion pin if needed
if outPin.name not in nodeInputPins:
self.onGraphInputPinCreated(outPin)
graphOutputNodes = self.rawGraph.getNodesList(
classNameFilters=['graphOutputs'])
graphOutputPins = {}
for graphOutputNode in graphOutputNodes:
for inPin in graphOutputNode.orderedInputs.values():
graphOutputPins[inPin.name] = inPin
# create companion pin if needed
if inPin.name not in nodeOutputPins:
self.onGraphOutputPinCreated(inPin)
for nodeInputPinName, nodeInputPin in nodeInputPins.items():
if nodeInputPinName not in graphInputPins:
if nodeInputPin in self.__inputsMap:
nodeInputPin.kill()
clearSignal(nodeInputPin.killed)
self.__inputsMap.pop(nodeInputPin)
for nodeOutputPinName, nodeOutputPin in nodeOutputPins.items():
if nodeOutputPinName not in graphOutputPins:
if nodeOutputPin in self.__outputsMap:
nodeOutputPin.kill()
clearSignal(nodeOutputPin.killed)
self.__outputsMap.pop(nodeOutputPin)
def Tick(self, delta):
self.syncPins()
self.rawGraph.Tick(delta)
super(compound, self).Tick(delta)
def setName(self, name):
super(compound, self).setName(name)
if self.rawGraph is not None:
self.rawGraph.name = self.getName()
@staticmethod
def category():
return 'SubGraphs'
@staticmethod
def keywords():
return []
@staticmethod
def description():
return 'Encapsulate a graph inside a node'
def serialize(self):
default = NodeBase.serialize(self)
default['graphData'] = self.rawGraph.serialize()
return default
def onGraphInputPinCreated(self, outPin):
"""Reaction when pin added to graphInputs node
:param outPin: output pin on graphInputs node
:type outPin: :class:`~PyFlow.Core.PinBase.PinBase`
"""
# add companion pin for graphInputs node's output pin
subgraphInputPin = self.createInputPin(outPin.name,
outPin.__class__.__name__,
outPin.defaultValue(),
outPin.call,
outPin.structureType,
outPin.constraint,
outPin.structConstraint,
group=outPin.owningNode().name)
if subgraphInputPin.isAny():
subgraphInputPin.supportedDataTypes = outPin.supportedDataTypes
subgraphInputPin.enableOptions(PinOptions.AllowAny
| PinOptions.DictElementSupported)
outPin.owningNode().constraints[outPin.constraint].append(
subgraphInputPin)
self.constraints[outPin.constraint].append(outPin)
outPin.owningNode().structConstraints[outPin.structConstraint].append(
subgraphInputPin)
self.structConstraints[outPin.structConstraint].append(outPin)
self.__inputsMap[subgraphInputPin] = outPin
pinAffects(subgraphInputPin, outPin)
# connect
def forceRename(name):
subgraphInputPin.setName(name, force=True)
outPin.nameChanged.connect(forceRename, weak=False)
# broadcast for UI wrapper class
self.pinExposed.send(subgraphInputPin)
def onGraphOutputPinCreated(self, inPin):
"""Reaction when pin added to graphOutputs node
:param inPin: input pin on graphOutputs node
:type inPin: :class:`~PyFlow.Core.PinBase.PinBase`
"""
# add companion pin for graphOutputs node's input pin
subgraphOutputPin = self.createOutputPin(inPin.name,
inPin.__class__.__name__,
inPin.defaultValue(),
inPin.structureType,
inPin.constraint,
inPin.structConstraint,
group=inPin.owningNode().name)
if subgraphOutputPin.isAny():
subgraphOutputPin.supportedDataTypes = inPin.supportedDataTypes
subgraphOutputPin.enableOptions(PinOptions.AllowAny
| PinOptions.DictElementSupported)
if subgraphOutputPin.isExec():
inPin.onExecute.connect(subgraphOutputPin.call)
inPin.owningNode().constraints[inPin.constraint].append(
subgraphOutputPin)
self.constraints[inPin.constraint].append(inPin)
inPin.owningNode().structConstraints[inPin.structConstraint].append(
subgraphOutputPin)
self.structConstraints[inPin.structConstraint].append(inPin)
self.__outputsMap[subgraphOutputPin] = inPin
pinAffects(inPin, subgraphOutputPin)
# connect
def forceRename(name):
subgraphOutputPin.setName(name, force=True)
inPin.nameChanged.connect(forceRename, weak=False)
# broadcast for UI wrapper class
self.pinExposed.send(subgraphOutputPin)
def kill(self, *args, **kwargs):
self.rawGraph.remove()
super(compound, self).kill(*args, **kwargs)
def postCreate(self, jsonTemplate=None):
super(compound, self).postCreate(jsonTemplate=jsonTemplate)
if jsonTemplate is not None and 'graphData' in jsonTemplate:
parentGraph = self.graph().graphManager.findGraph(
jsonTemplate['owningGraphName'])
self.rawGraph = GraphBase(self.name,
self.graph().graphManager, parentGraph)
# recreate graph contents
jsonTemplate['graphData']['name'] = self.getName()
self.rawGraph.populateFromJson(jsonTemplate['graphData'])
self.syncPins()
inputsMap = self.namePinInputsMap
for inpJson in jsonTemplate['inputs']:
inputsMap[inpJson['name']].uid = uuid.UUID(inpJson['uuid'])
outputsMap = self.namePinOutputsMap
for outJson in jsonTemplate['outputs']:
outputsMap[outJson['name']].uid = uuid.UUID(outJson['uuid'])
else:
self.rawGraph = GraphBase(self.name,
self.graph().graphManager,
self.graph().graphManager.activeGraph())
def addNode(self, node):
self.rawGraph.addNode(node)
def autoAffectPins(self):
pass
def compute(self, *args, **kwargs):
# put data from inner graph pins to outer compound node output companions
for outputPin, innerPin in self.__outputsMap.items():
outputPin.setData(innerPin.getData())
outputPin.setClean()
class AnyPin(PinBase):
"""**Abstract Pin -- "AnyPin"**
This Pin Type is an abstraction of Pins, it is a Pin that will act as any other defined Pin.
This type of Pin allow to create abstract Nodes that can operate in more than one dataType.
By default AnyPin non Initialized will be marked as error, as Pyflow can't know what is inside.
This Error can be avoided by enabling :py:attr:`PyFlow.Core.Common.PinOptions.AllowAny`. Thas how NonTyped Lists are made.
By default :py:attr:`PyFlow.Core.Common.PinOptions.ChangeTypeOnConnection` is enabled, and that means that it will change
its internal dataType to the new dataType provided by connection or user Initialization. If disabled, pin will not allow changes.
Is important to define a bunch of allowedDataTypes on pin creation, this will restrict what pins can be connected and what no,
so even being a AnyPin, it can be defined to allow for example only ["FloatPin","IntPin"] so only those could be connected.
:param self.singleInit: can be set to True, so once initialized, it will never be able to change dataType
:param self.checkForErrors: can be set To False so it will never try to find errors
Signals:
* **typeChanged** : Fired when dataType has change
"""
def __init__(self, name, owningNode, direction, **kwargs):
"""
:param name: Pin name
:type name: string
:param owningNode: Owning Node
:type owningNode: :py:class:`PyFlow.Core.NodeBase.NodeBase`
:param direction: PinDirection , can be input or output
:type direction: :py:class:`PyFlow.Core.Common.PinDirection`
"""
super(AnyPin, self).__init__(name, owningNode, direction, **kwargs)
self.typeChanged = Signal(str)
self.dataTypeBeenSet = Signal()
self.setDefaultValue(None)
self._isAny = True
# if True, setType and setDefault will work only once
self.singleInit = False
self.checkForErrors = True
self.enableOptions(PinOptions.ChangeTypeOnConnection)
self._defaultSupportedDataTypes = self._supportedDataTypes = tuple([pin.__name__ for pin in getAllPinClasses() if pin.IsValuePin()])
self.canChange = True
self.super = None
self.prevDataType = None
self._lastError2 = None
@PinBase.dataType.getter
def dataType(self):
return self.activeDataType
@staticmethod
def supportedDataTypes():
"""Tuple with all the Defined value Pin Classes
"""
return tuple([pin.__name__ for pin in getAllPinClasses() if pin.IsValuePin()])
@staticmethod
def IsValuePin():
return True
@staticmethod
def defColor():
return (200, 200, 200, 255)
@staticmethod
def color():
return (200, 200, 200, 255)
@staticmethod
def pinDataTypeHint():
return 'AnyPin', None
@staticmethod
def internalDataStructure():
return type(None)
@staticmethod
def processData(data):
return data
def enableOptions(self, *options):
super(AnyPin, self).enableOptions(*options)
if not self.optionEnabled(PinOptions.ChangeTypeOnConnection):
self.super = AnyPin
self.updateError([])
def disableOptions(self, *options):
super(AnyPin, self).disableOptions(*options)
if not self.optionEnabled(PinOptions.ChangeTypeOnConnection):
self.super = AnyPin
self.updateError([])
def setTypeFromData(self, data):
"""Initialize DataType from actual data
Iterates all defined Pin and compares type(data) with Pin.internalDataStructure() to find a valid DataType
:param data: Actual data to search Pin/dataType from
"""
for pin in [pin for pin in getAllPinClasses() if pin.IsValuePin()]:
pType = pin.internalDataStructure()
if type(data) == pType:
if pin.__name__ != self.activeDataType:
if self.optionEnabled(PinOptions.ChangeTypeOnConnection):
traverseConstrainedPins(self, lambda x: self.updateOnConnectionCallback(x, pin.__name__, True, None))
self.owningNode().checkForErrors()
break
def updateError(self, traversed=[], updateNeis=False):
"""Check is Pin dataType is "AnyPin" and if it is, checks if it can change Type on conection, and if it can, marked as error.
Is a iterative Function that traverses conected and constrained Pins
:param traversed: Current Iterated neighbours, defaults to []
:type traversed: list, optional
:param updateNeis: Try to update Constrained Pins parents error display, it can be slow so use carefully, defaults to False
:type updateNeis: bool, optional
"""
if not self.checkForErrors:
return
nodePins = set([self])
if self.constraint:
nodePins = set(self.owningNode().constraints[self.constraint])
for connectedPin in getConnectedPins(self):
if connectedPin.isAny():
nodePins.add(connectedPin)
for neighbor in nodePins:
if neighbor not in traversed:
if all([neighbor.activeDataType == "AnyPin",
neighbor.canChangeTypeOnConection([], neighbor.optionEnabled(PinOptions.ChangeTypeOnConnection), []) or not neighbor.optionEnabled(PinOptions.AllowAny)]) :
neighbor.setError("AnyPin Not Initialized")
neighbor.super = None
else:
neighbor.clearError()
if neighbor.activeDataType == "AnyPin":
neighbor.super = AnyPin
traversed.append(neighbor)
if neighbor.isAny():
neighbor.updateError(traversed, updateNeis)
if updateNeis:
neighbor.owningNode().checkForErrors()
def serialize(self):
"""Stores The data to Json
Appends current value and currentDataType to default :py:func:`PyFlow.Core.PinBase.PinBase.serialize` method
:returns: json data
:rtype: {dict}
"""
dt = super(AnyPin, self).serialize()
constrainedType = self.activeDataType
if constrainedType != self.__class__.__name__:
pinClass = findPinClassByType(constrainedType)
# serialize with active type's encoder
dt['value'] = json.dumps(self.currentData(), cls=pinClass.jsonEncoderClass())
dt['currDataType'] = constrainedType
return dt
def deserialize(self, jsonData):
"""Reconstruct Pin from saved jsonData
:param jsonData: Input Json Saved data
:type jsonData: dict
"""
super(AnyPin, self).deserialize(jsonData)
if "currDataType" in jsonData:
self.setType(jsonData["currDataType"])
pinClass = findPinClassByType(self.activeDataType)
try:
self.setData(json.loads(jsonData['value'], cls=pinClass.jsonDecoderClass()))
except:
self.setData(self.defaultValue())
self.updateError([])
def pinConnected(self, other):
"""Pin Connection been Made
We update Error here to search for nonInitialized Pins in current Node, and in connected Nodes if initializing
:param other: Pin that has been conected to this Pin.
:type other: :py:class:`PyFlow.Core.PinBase.PinBase`
"""
super(AnyPin, self).pinConnected(other)
self._lastError2 = self._lastError
self.updateError([],self.activeDataType == "AnyPin" or self.prevDataType == "AnyPin")
self.owningNode().checkForErrors()
def aboutToConnect(self, other):
"""Function called before real connection but after :py:func:`PyFlow.Core.Common.canConnectPins` returns True
We traverse conected and constrained Pins here to search if we can change Pin dataType, and if we can we traverse again
changing all the necesary datatypes in conected Graph Pins.
:param other: Pin that will be conected to this Pin.
:type other: :py:class:`PyFlow.Core.PinBase.PinBase`
"""
if self.canChangeTypeOnConection([], self.optionEnabled(PinOptions.ChangeTypeOnConnection), []):
dataType = other.dataType
traverseConstrainedPins(self, lambda pin: self.updateOnConnectionCallback(pin, dataType, False, other))
super(AnyPin, self).aboutToConnect(other)
def pinDisconnected(self, other):
"""Pin has been disconnected
We update error here and checkFor errors in owning Node
:param other: Pin that has been disconected to this Pin.
:type other: :py:class:`PyFlow.Core.PinBase.PinBase`
"""
super(AnyPin, self).pinDisconnected(other)
self.updateError([],self.activeDataType == "AnyPin" or self.prevDataType == "AnyPin")
self._lastError2 = self._lastError
if self.activeDataType == "AnyPin" and self._lastError2 == None:
self.prevDataType = "AnyPin"
else:
self.prevDataType = None
self.owningNode().checkForErrors()
def updateOnConnectionCallback(self, pin, dataType, init=False, other=None):
"""Method Called in traverse function :py:func:`PyFlow.Core.Common.traverseConstrainedPins`
This Function is called for all the conected Pins to the initial Pin calling it.
Here we traverse all pins and call :py:func:`AnyPin.setType` for all of them.
We also intersect all the conected pins allowedDataTypes.
:param pin: Pin to perform operations on
:type pin: :py:class:`AnyPin`
:param dataType: New DataType to apply
:type dataType: string
:param init: If initializing AnyPin can have same strenght as other types, if not, "AnyPin" Pin will always be weaker than other dataType, if, defaults to False
:type init: bool, optional
:param other: other Pin to heredate stuff from him, defaults to None
:type other: :py:class:`PyFlow.Core.PinBase.PinBase`, optional
"""
free = pin.checkFree([])
if free:
if (dataType == "AnyPin" and not init):
if not other:
return
else:
if pin.dataType != "AnyPin" and pin.dataType in other.allowedDataTypes([], other._supportedDataTypes) and other.canChangeTypeOnConection([], other.optionEnabled(PinOptions.ChangeTypeOnConnection), []):
dataType = pin.dataType
if any([dataType in pin.allowedDataTypes([], pin._supportedDataTypes),
dataType == "AnyPin",
(pin.checkFree([], False) and dataType in pin.allowedDataTypes([], pin._defaultSupportedDataTypes, defaults=True))]):
a = pin.setType(dataType)
if a:
if other:
if pin.optionEnabled(PinOptions.ChangeTypeOnConnection):
pin._supportedDataTypes = other.allowedDataTypes([], other._supportedDataTypes)
if dataType == "AnyPin":
if pin.optionEnabled(PinOptions.ChangeTypeOnConnection):
pin._supportedDataTypes = pin._defaultSupportedDataTypes
pin.supportedDataTypes = lambda: pin._supportedDataTypes
def checkFree(self, checked=[], selfChek=True):
"""Recursive Function to find if all connected Pins are of type :py:class:`AnyPin` and canChange On conection,
so basically it checks if a Pin is free to change its dataType to another one
:param checked: Already visited Pins, defaults to []
:type checked: list, optional
:param selfChek: Define if check Pin itself or no, this is useful when trying to override a conection that is in fact
the only conection that make hole graphed nodes not be able to change Type, defaults to True
:type selfChek: bool, optional
:returns: True if Pin can change current dataType
:rtype: {bool}
"""
if self.constraint is None or self.dataType == self.__class__.__name__:
return True
else:
con = []
if selfChek:
free = not self.hasConnections()
if not free:
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
else:
free = True
checked.append(self)
canChange = self.canChangeTypeOnConection([], self.optionEnabled(PinOptions.ChangeTypeOnConnection), [])
free = canChange
for port in self.owningNode().constraints[self.constraint] + con:
if port not in checked:
checked.append(port)
if not isinstance(port, AnyPin):
free = False
elif free:
free = port.checkFree(checked)
return free
def allowedDataTypes(self, checked=[], dataTypes=[], selfChek=True, defaults=False):
"""Recursive Function to intersect allowedDatatypes of all conected pins.
:param checked: Already visited Pins, defaults to []
:type checked: list, optional
:param dataTypes: Intersected dataTypes, defaults to []
:type dataTypes: list, optional
:param selfChek: Define if check Pin itself or no, this is useful when trying to override a conection that is in fact
the only conection that make hole graphed nodes not be able to change Type, defaults to True
:type selfChek: bool, optional
:param defaults: Define if we are intersecting current allowedDataTypes, or default (as in definition of node) allowedDataTypes, defaults to False
:type defaults: bool, optional
:returns: List contatining all the intersected dataTypes
:rtype: {list}
"""
if not self.optionEnabled(PinOptions.ChangeTypeOnConnection) and self.activeDataType == "AnyPin":
return self._defaultSupportedDataTypes
con = []
neis = []
if selfChek:
if self.hasConnections():
for c in getConnectedPins(self):
if c not in checked:
con.append(c)
else:
checked.append(self)
if self.constraint:
neis = self.owningNode().constraints[self.constraint]
for port in neis + con:
if port not in checked:
checked.append(port)
if not defaults:
dataTypes = list(set(dataTypes) & set(port._supportedDataTypes))
else:
dataTypes = list(set(dataTypes) & set(port._defaultSupportedDataTypes))
dataTypes = port.allowedDataTypes(checked, dataTypes, selfChek=True, defaults=defaults)
return dataTypes
def initType(self, dataType, initializing=False):
"""Same as :py:func:`AnyPin.aboutToConnect` but instead of using another Pin using a dataType name
:param dataType: New DataType we want the pin to became
:type dataType: string
:param initializing: If initializing AnyPin can have same strenght as other types, if not, "AnyPin" Pin will always be weaker than other dataType, if, defaults to False
:type initializing: bool, optional
:returns: True if it can change to the asked dataType
:rtype: {bool}
"""
if self.canChangeTypeOnConection([], self.optionEnabled(PinOptions.ChangeTypeOnConnection), []):
traverseConstrainedPins(self, lambda pin: self.updateOnConnectionCallback(pin, dataType, initializing))
self._lastError2 = self._lastError
self.updateError([],self.activeDataType == "AnyPin" or self.prevDataType == "AnyPin")
self.owningNode().checkForErrors()
return True
return False
def setType(self, dataType):
"""Here is where :py:class:`AnyPin` heredates all the properties from other defined dataTypes and act like those
:param dataType: New DataType
:type dataType: string
:returns: True if succes setting dataType
:rtype: {bool}
"""
if self.activeDataType == dataType:
return True
if not self.optionEnabled(PinOptions.ChangeTypeOnConnection):
return False
if self.activeDataType != self.__class__.__name__ and self.singleInit:
# Marked as single init. Type already been set. Skip
return False
otherClass = findPinClassByType(dataType)
if dataType != "AnyPin":
self.super = otherClass
else:
self.super = None
if self.activeDataType == "AnyPin" and self._lastError2 == None:
self.prevDataType = "AnyPin"
else:
self.prevDataType = None
self.activeDataType = dataType
if not self.isArray():
self.setData(getPinDefaultValueByType(self.activeDataType))
else:
self.setData([])
self.setDefaultValue(self._data)
self.color = otherClass.color
self.dirty = True
self.jsonEncoderClass = otherClass.jsonEncoderClass
self.jsonDecoderClass = otherClass.jsonDecoderClass
self.supportedDataTypes = otherClass.supportedDataTypes
self._supportedDataTypes = otherClass.supportedDataTypes()
self.typeChanged.send(self.activeDataType)
self.dataBeenSet.send(self)
return True
class EditorHistory(object):
"""docstring for EditorHistory."""
def __init__(self, app):
self.statePushed = Signal(object)
self.stateRemoved = Signal(object)
self.stateSelected = Signal(object)
self.app = app
self.stack = list()
try:
self._capacity = int(ConfigManager().getPrefsValue("PREFS", "General/HistoryDepth"))
except:
self._capacity = 10
self.activeState = None
def shutdown(self):
clearSignal(self.statePushed)
clearSignal(self.stateRemoved)
clearSignal(self.stateSelected)
clearList(self.stack)
def getStack(self):
return self.stack
def count(self):
return len(self.stack)
@property
def capacity(self):
return self._capacity
@capacity.setter
def capacity(self, value):
self._capacity = value
if value < len(self.stack):
for i in range(len(self.stack) - value):
state = self.stack.pop()
self.stateRemoved.send(state)
def clear(self):
clearList(self.stack)
def stateIndex(self, state):
if state in self.stack:
return self.stack.index(state)
return -1
@property
def currentIndex(self):
if self.activeState is not None:
return self.stateIndex(self.activeState)
return -1
def push(self, edState):
if self.currentIndex < self.count() - 1:
nextState = None
while True:
index = self.count() - 1
nextState = self.stack[index]
if nextState == self.activeState:
break
state = self.stack.pop()
self.stateRemoved.send(state)
self.stack.append(edState)
if len(self.stack) >= self.capacity:
poppedState = self.stack.pop(0)
self.stateRemoved.send(poppedState)
self.statePushed.send(edState)
self.activeState = edState
self.stateSelected.send(edState)
def selectState(self, state):
for st in self.stack:
if state == st:
self.app.loadFromData(st.editorState)
self.activeState = st
self.stateSelected.send(st)
break
def select(self, index):
index = clamp(index, 0, self.count() - 1)
if index == self.currentIndex:
return
if len(self.stack) == 0:
return
stateData = self.stack[index].editorState
self.app.loadFromData(stateData)
state = self.stack[index]
self.activeState = state
self.stateSelected.send(state)
def saveState(self, text):
self.push(_EditorState(text))
def undo(self):
if self.currentIndex > 0:
self.select(self.currentIndex - 1)
def redo(self):
self.select(self.currentIndex + 1)
class TransmissionRPC():
"""
Low-level AsyncIO Transmission RPC communication
This class handles connecting to a Transmission daemon via the RPC
interface. It does not implement the RPC protocol, only basic things like
authentication, sending requests and receiving responses. High-level RPCs
are done in the *API classes.
"""
def __init__(self,
host='localhost',
port=9091,
*,
tls=False,
user='',
password='',
proxy='',
path='/transmission/rpc',
enabled=True):
self.host = host
self.port = port
self.path = path
self.tls = tls
self.user = user
self.password = password
self.proxy = proxy
self._headers = {'content-type': 'application/json'}
self._session = None
self._enabled_event = asyncio.Event()
self.enabled = enabled
self._request_lock = asyncio.Lock()
self._connecting_lock = asyncio.Lock()
self._connection_tested = False
self._connection_exception = None
self._timeout = TIMEOUT
self._version = None
self._rpcversion = None
self._rpcversionmin = None
self._on_connecting = Signal()
self._on_connected = Signal()
self._on_disconnected = Signal()
self._on_error = Signal()
def on(self, signal, callback, autoremove=True):
"""
Register `callback` for `signal`
signal: 'connecting', 'connected', 'disconnected' or 'error'
callback: a callable that receives this instance as a positional
argument and, in case of the 'error' signal, the exception as
a keyword argument with the name 'error'
Callbacks are automatically unsubscribed when they are
garbage-collected.
"""
try:
sig = getattr(self, '_on_' + signal)
except AttributeError:
raise ValueError('Unknown signal: {!r}'.format(signal))
else:
if not isinstance(sig, Signal):
raise ValueError('Unknown signal: {!r}'.format(signal))
else:
log.debug('Registering %r for %r event', callback, signal)
sig.connect(callback, weak=autoremove)
@property
def version(self):
"""Version of the Transmission daemon or None if not connected"""
return self._version
@property
def rpcversion(self):
"""RPC version of the Transmission daemon or None if not connected"""
return self._rpcversion
@property
def rpcversionmin(self):
"""Oldest RPC version supported by Transmission daemon or None if not connected"""
return self._rpcversionmin
@property
def host(self):
"""
Hostname or IP of the Transmission RPC interface
Setting this property calls disconnect().
"""
return self._host
@host.setter
def host(self, host):
self._host = str(host) if host is not None else 'localhost'
asyncio.ensure_future(self.disconnect('Changing host: %r' %
self._host))
@property
def path(self):
"""
Path of the Transmission RPC interface
Setting this property calls disconnect().
"""
return self._path
@path.setter
def path(self, path):
if path is None:
path = '/transmission/rpc'
elif not path or path[0] != '/':
path = '/' + path
self._path = path
asyncio.ensure_future(self.disconnect('Changing path: %r' %
self._path))
@property
def port(self):
"""
Port of the Transmission RPC interface
Setting this property calls disconnect().
"""
return self._port
@port.setter
def port(self, port):
self._port = int(port) if port is not None else 9091
asyncio.ensure_future(self.disconnect('Changing port: %r' %
self._port))
@property
def user(self):
"""
Username for authenticating to the Transmission RPC interface or empty string
Setting this property calls disconnect().
"""
return self._user
@user.setter
def user(self, user):
self._user = str(user) if user is not None else ''
asyncio.ensure_future(self.disconnect('Changing user: %r' %
self._user))
@property
def password(self):
"""
Password for authenticating to the Transmission RPC interface or empty string
Setting this property calls disconnect().
"""
return self._password
@password.setter
def password(self, password):
self._password = str(password) if password is not None else ''
asyncio.ensure_future(
self.disconnect('Changing password: %r' % self._password))
@property
def tls(self):
"""
Whether to use HTTPS for connecting to the Transmission RPC interface
Setting this property calls disconnect().
"""
return self._tls
@tls.setter
def tls(self, tls):
self._tls = bool(tls) if tls is not None else False
asyncio.ensure_future(self.disconnect('Changing tls: %r' % self._tls))
@property
def url(self):
"""
URL of the Transmission RPC interface
Setting or getting this property sets or gets the following properties: tls, user,
password, host, port, path
Missing parts are filled in with defaults, e.g. "example.org:1234" results in
"http://example.org:1234/transmission/rpc".
While user and password are supported when setting ("user:password@localhost"),
they are not included in the return value for security reasons. See the
url_unsafe property.
"""
return '%s://%s:%d%s' % ('https' if self.tls else 'http', self.host,
self.port, self.path)
@url.setter
def url(self, url):
if url is None:
url = URL('http://localhost:9091/transmission/rpc')
else:
url = URL(url)
self._user = url.user or ''
self._password = url.password or ''
if url.scheme == 'https':
self._tls = True
elif url.scheme == 'http':
self._tls = False
else:
raise ValueError('Invalid scheme: %r' % (url.scheme, ))
self._host = url.host
self._port = int(url.port) if url.port is not None else 9091
self._path = url.path if url.path is not None else '/transmission/rpc'
asyncio.ensure_future(self.disconnect('Changing url: %r' % self._url))
@property
def url_unsafe(self):
"""URL of the Transmission RPC interface with user and password if given"""
if self.user or self.password:
return '%s://%s:%s@%s:%d%s' % ('https' if self.tls else 'http',
self.user, self.password, self.host,
self.port, self.path)
else:
return self.url
@property
def proxy(self):
"""URL of a SOCKS5, SOCKS4 or HTTP proxy"""
return self._proxy
@proxy.setter
def proxy(self, proxy):
if proxy:
try:
import aiohttp_socks
except ImportError:
raise ValueError('Missing extra: proxy (aiohttp_socks)')
self._connector = aiohttp_socks.ProxyConnector.from_url(proxy)
self._proxy = URL(proxy)
else:
self._connector = None
self._proxy = URL('')
asyncio.ensure_future(
self.disconnect('Changing proxy: %r' % self._proxy))
@property
def timeout(self):
"""Number of seconds to try to connect before giving up"""
return self._timeout
@timeout.setter
def timeout(self, timeout):
self._timeout = float(timeout)
@property
def enabled(self):
"""
Whether requests should connect
If this is set to False, requests will wait for it to be set to True.
This allows you to block any connection attempts until the connection
parameters (host, user, password, etc) are specified to prevent any
unwarranted error messages.
"""
return self._enabled_event.is_set()
@enabled.setter
def enabled(self, enabled):
if enabled and not self.enabled:
log.debug('Enabling %r', self)
self._enabled_event.set()
elif not enabled and self.enabled:
log.debug('Disabling %r', self)
self._enabled_event.clear()
if self.connected:
asyncio.ensure_future(self.disconnect())
@property
def connected(self):
"""Return True if connected, False otherwise"""
return (self._session is not None and not self._session.closed
and self._connection_tested)
async def connect(self):
"""
Connect to running daemon
If the `enabled` property is set to False, this method blocks until
`enabled` is set to True.
Raises RPCError, ConnectionError or AuthError.
"""
log.debug('Connecting to %s (timeout=%ss)', self.url, self.timeout)
self._on_connecting.send(self)
if self._connecting_lock.locked():
if self._connection_exception is not None:
# The other connect() call failed
log.debug('Found connection error: %r',
self._connection_exception)
raise self._connection_exception
log.debug('Connection is already being established - Waiting ...')
try:
async with async_timeout.timeout(self.timeout):
await self._enabled_event.wait()
except asyncio.TimeoutError:
raise TimeoutError(self.timeout, self.url)
else:
if self.connected:
log.debug('Connection is up: %r', self.url)
return
async with self._connecting_lock:
log.debug('Acquired connect() lock')
if self.connected:
await self.disconnect('Reconnecting')
# Block until we're enabled
await self._enabled_event.wait()
import aiohttp
session_args = {}
if self.user or self.password:
session_args['auth'] = aiohttp.BasicAuth(self.user,
self.password,
encoding='utf-8')
if self._connector is not None:
session_args['connector'] = self._connector
session_args['connector_owner'] = False
self._session = aiohttp.ClientSession(**session_args)
# Check if connection works
log.debug('Testing connection to %s', self.url)
try:
test_request = json.dumps({'method': 'session-get'})
info = await self._send_request(test_request)
except ClientError as e:
self._connection_exception = e
log.debug('Caught during connection test: %r', e)
await self._reset()
self._on_error.send(self, error=e)
raise
else:
self._version = info['version']
self._rpcversion = info['rpc-version']
self._rpcversionmin = info['rpc-version-minimum']
self._connection_tested = True
self._connection_exception = None
log.debug('Connection established: %s', self.url)
self._on_connected.send(self)
log.debug('Releasing connect() lock')
async def disconnect(self, reason=None):
"""
Disconnect if connected
reason: Why are we disconnecting? Only used in a debugging message.
"""
if self.connected:
await self._reset()
log.debug('Disconnecting from %s (%s)', self.url,
reason if reason is not None else 'for no reason')
self._on_disconnected.send(self)
async def _reset(self):
if self._session is not None:
await self._session.close()
self._session = None
self._version = None
self._rpcversion = None
self._rpcversionmin = None
self._connection_tested = False
async def _post(self, data):
async with async_timeout.timeout(self.timeout):
response = await self._session.post(self.url,
data=data,
headers=self._headers)
if response.status == CSRF_ERROR_CODE:
# Send request again with CSRF header
self._headers[CSRF_HEADER] = response.headers[CSRF_HEADER]
log.debug('Setting CSRF header: %s = %s', CSRF_HEADER,
response.headers[CSRF_HEADER])
await response.release()
return await self._post(data)
elif response.status == AUTH_ERROR_CODE:
await response.release()
log.debug('Authentication failed: %s: user=%r, password=%r',
self.url, self.user, self.password)
raise AuthError(self.url)
else:
try:
answer = await response.json()
except json.JSONDecodeError as e:
raise RPCError('Server sent malformed JSON: %s: %s' %
(e, await response.text()))
else:
return answer
async def _send_request(self, post_data):
"""
Send RPC POST request to daemon
post_data: Any valid RPC request as JSON string
If applicable, returns response['arguments']['torrents'] or
response['arguments'], otherwise response.
Raises ClientError.
"""
# NOTE #163: Letting asyncio.CancelledError bubble up seems to fix the issue that
# causes empty torrent lists in new tabs until the next poll iteration.
# But I've seen this error pop up in the TUI log: "Unclosed client
# session client_session: <aiohttp.client.ClientSession object at
# 0x7f35d98d1be0>" This may or may not be related.
import aiohttp
try:
from aiohttp_socks import ProxyConnectionError, ProxyError, ProxyTimeoutError
except ImportError:
class ProxyError(Exception):
pass
class ProxyConnectionError(Exception):
pass
class ProxyTimeoutError(Exception):
pass
try:
answer = await self._post(post_data)
except aiohttp.ClientError as e:
log.debug('Caught during POST request: %r', e)
raise ConnectionError(self.url)
except (ProxyError, ProxyConnectionError) as e:
log.debug('Caught during POST request: %r', e)
raise ConnectionError(self.proxy)
except asyncio.TimeoutError as e:
log.debug('Caught during POST request: %r', e)
raise TimeoutError(self.timeout, self.url)
except ProxyTimeoutError as e:
log.debug('Caught during POST request: %r', e)
raise TimeoutError(self.timeout, self.proxy)
else:
if answer['result'] != 'success':
raise RPCError(answer['result'].capitalize())
else:
if 'arguments' in answer:
if 'torrents' in answer['arguments']:
return answer['arguments']['torrents']
else:
return answer['arguments']
return answer
def __getattr__(self, method):
"""
Return asyncio coroutine that sends RPC request and returns response
method: Any method from the RPC specs with every '-' replaced with '_'.
For arguments see the RPC specs.
Example:
>>> stats = await client.session_stats()
>>> torrents = await client.torrent_get(ids=(1,2,3), fields=('status','name'))
Raises RPCError, ConnectionError, AuthError
"""
async def request(arguments=None, **kwargs):
arguments = arguments or {}
async with self._request_lock:
if not self.connected:
log.debug('Autoconnecting for %r', method)
await self.connect()
arguments.update(**kwargs)
data = {
'method': method.replace('_', '-'),
'arguments': arguments
}
try:
rpc_request = json.dumps(data)
except Exception as e:
raise RuntimeError('Invalid JSON data: %s: %r' %
(e, data)) from None
try:
return await self._send_request(rpc_request)
except ClientError as e:
log.debug('Caught ClientError in %r request: %r', method,
e)
# RPCError does not mean host is unreachable, there was just a
# misunderstanding, so we're still connected.
if not isinstance(e, RPCError) and self.connected:
await self.disconnect(str(e))
self._on_error.send(self, error=e)
raise
request.__name__ = method
request.__qualname__ = method
return request
class GraphManager(object):
"""docstring for GraphManager."""
def __init__(self):
super(GraphManager, self).__init__()
self.graphChanged = Signal(object)
self._graphs = {}
self._activeGraph = None
self._activeGraph = GraphBase(ROOT_GRAPH_NAME, self)
self._activeGraph.setIsRoot(True)
def findRootGraph(self):
roots = []
for graph in self.getAllGraphs():
if graph.isRoot():
roots.append(graph)
assert (len(roots) == 1), "Fatal! Multiple roots!"
return roots[0]
def selectRootGraph(self):
self.selectGraph(self.findRootGraph())
def serialize(self):
rootGraph = self.findRootGraph()
saved = rootGraph.serialize()
saved["fileVersion"] = str(version.currentVersion())
saved["activeGraph"] = self.activeGraph().name
return saved
@dispatch(str)
def removeGraph(self, name):
graph = self.findGraph(name)
if graph is not None:
graph.clear()
self._graphs.pop(graph.uid)
if graph.parentGraph is not None:
if graph in graph.parentGraph.childGraphs:
graph.parentGraph.childGraphs.remove(graph)
del graph
@dispatch(object)
def removeGraph(self, graph):
if graph.uid in self._graphs:
graph.clear()
self._graphs.pop(graph.uid)
if graph.parentGraph is not None:
if graph in graph.parentGraph.childGraphs:
graph.parentGraph.childGraphs.remove(graph)
del graph
def deserialize(self, data):
if "fileVersion" in data:
fileVersion = version.Version.fromString(data["fileVersion"])
else:
# handle older version
pass
self.clear(keepRoot=False)
self._activeGraph = GraphBase(str('root'), self)
self._activeGraph.populateFromJson(data)
self._activeGraph.setIsRoot(True)
self.selectGraph(self._activeGraph)
def clear(self, keepRoot=True, *args, **kwargs):
self.selectGraph(ROOT_GRAPH_NAME)
self.removeGraph(ROOT_GRAPH_NAME)
self._graphs.clear()
self._graphs = {}
del self._activeGraph
self._activeGraph = None
if keepRoot:
self._activeGraph = GraphBase(ROOT_GRAPH_NAME, self)
self.selectGraph(self._activeGraph)
self._activeGraph.setIsRoot(True)
def Tick(self, deltaTime):
for graph in self._graphs.values():
graph.Tick(deltaTime)
def findVariableRefs(self, variable):
result = []
for node in self.getAllNodes(classNameFilters=['getVar', 'setVar']):
if node.variableUid() == variable.uid:
result.append(node)
return result
@dispatch(str)
def findGraph(self, name):
graphs = self.graphsDict
if name in graphs:
return graphs[name]
return None
def findPinByName(self, pinFullName):
result = None
for graph in self.getAllGraphs():
result = graph.findPin(pinFullName)
if result is not None:
break
return result
@dispatch(str)
def findNode(self, name):
"""Finds a node across all graphs
"""
result = None
for graph in self.getAllGraphs():
result = graph.findNode(name)
if result is not None:
break
return result
@dispatch(uuid.UUID)
def findNode(self, uid):
"""Finds a node across all graphs
"""
for graph in self.getAllGraphs():
if uid in graph.nodes:
return graph.nodes[uid]
return None
@dispatch(uuid.UUID)
def findVariable(self, uuid):
"""Finds a variable across all graphs
"""
result = None
for graph in self._graphs.values():
if uuid in graph.vars:
result = graph.vars[uuid]
break
return result
@dispatch(str)
def findVariable(self, name):
"""Finds a variable across all graphs
"""
for graph in self._graphs.values():
for var in graph.vars.values():
if var.name == name:
return var
return None
def location(self):
location = [self.activeGraph().name]
parent = self.activeGraph().parentGraph
while parent is not None:
location.insert(0, parent.name)
parent = parent.parentGraph
return location
@property
def graphsDict(self):
result = {}
for graph in self.getAllGraphs():
result[graph.name] = graph
return result
def add(self, graph):
graph.name = self.getUniqGraphName(graph.name)
self._graphs[graph.uid] = graph
def activeGraph(self):
return self._activeGraph
@dispatch(str)
def selectGraph(self, name):
graphs = self.graphsDict
if name in graphs:
if name != self.activeGraph().name:
oldGraph = self.activeGraph()
newGraph = graphs[name]
self._activeGraph = newGraph
self.graphChanged.send(self.activeGraph())
@dispatch(object)
def selectGraph(self, graph):
for newGraph in self.getAllGraphs():
if newGraph.name == graph.name:
if newGraph.name != self.activeGraph().name:
oldGraph = self.activeGraph()
self._activeGraph = newGraph
self.graphChanged.send(self.activeGraph())
break
def getAllGraphs(self):
return [g for g in self._graphs.values()]
def getAllNodes(self, classNameFilters=[]):
allNodes = []
for graph in self.getAllGraphs():
if len(classNameFilters) == 0:
allNodes.extend(list(graph.nodes.values()))
else:
allNodes.extend([
node for node in graph.nodes.values()
if node.__class__.__name__ in classNameFilters
])
return allNodes
def getAllVariables(self):
result = []
for graph in self.getAllGraphs():
result.extend(list(graph.vars.values()))
return result
def getUniqGraphPinName(self, graph, name):
existingNames = []
for node in graph.getNodes(
classNameFilters=['graphInputs', 'graphOutputs']):
existingNames.extend([pin.name for pin in node.pins])
return getUniqNameFromList(existingNames, name)
def getUniqPinName(self, name):
existingNames = []
for node in self.getAllNodes():
existingNames.extend([pin.name for pin in node.pins])
return getUniqNameFromList(existingNames, name)
def getAllNames(self):
existingNames = [g.name for g in self.getAllGraphs()]
existingNames.extend([n.name for n in self.getAllNodes()])
existingNames.extend([var.name for var in self.getAllVariables()])
for node in self.getAllNodes():
existingNames.extend([pin.name for pin in node.pins])
return existingNames
def getUniqName(self, name):
existingNames = self.getAllNames()
return getUniqNameFromList(existingNames, name)
def getUniqGraphName(self, name):
existingNames = [g.name for g in self.getAllGraphs()]
return getUniqNameFromList(existingNames, name)
def getUniqNodeName(self, name):
existingNames = [n.name for n in self.getAllNodes()]
if name in existingNames:
existingNames.remove(name)
return getUniqNameFromList(existingNames, name)
def getUniqVariableName(self, name):
existingNames = [var.name for var in self.getAllVariables()]
return getUniqNameFromList(existingNames, name)
def plot(self):
root = self.findRootGraph()
print("Active graph: {0}".format(str(self.activeGraph().name)),
"All graphs:", [g.name for g in self._graphs.values()])
root.plot()
