class SubjectiveFileDialog(QWidget):
def __init__(self, *args, single=True, dialog_root=None):
if dialog_root is None:
dialog_root = os.getcwd()
super().__init__(*args)
self.dialog_root = dialog_root
self.subject = BehaviorSubject(None)
layout = QHBoxLayout()
self.btn = SubjectivePushButton('Open')
if single:
self.btn.subject.subscribe(on_next=lambda _: self.get_file())
else:
self.btn.subject.subscribe(on_next=lambda _: self.get_files())
layout.addWidget(self.btn)
self.setLayout(layout)
def get_file(self):
filename = QFileDialog.getOpenFileName(self, 'Open File',
self.dialog_root)
self.subject.on_next(filename[0])
def get_files(self):
dialog = QFileDialog()
dialog.setFileMode(QFileDialog.AnyFile)
if dialog.exec_():
filenames = dialog.selectedFiles()
self.subject.on_next(filenames)
class SensorController:
rx_sensor_data_subject = None
rx_sensor_settings_subject = None
data_thread = None
def __init__(self):
self.rx_sensor_data_subject = Subject()
self.rx_sensor_settings_subject = BehaviorSubject(SensorSettings())
def start_data(self):
if self.data_thread is None or not self.data_thread.data_running:
self.data_thread = DataThread(self.rx_sensor_data_subject,
self.rx_sensor_settings_subject)
self.data_thread.start()
def stop_data(self):
if self.data_thread is not None:
self.data_thread.stop()
self.data_thread = None
def sensor_connected(self):
return self.data_thread is not None
def update_sensor_settings(self, sensor_settings: SensorSettings):
self.rx_sensor_settings_subject.on_next(sensor_settings)
def get_sensor_settings(self):
return self.rx_sensor_settings_subject.value
def list_serial_ports(self):
return [
comport.device for comport in serial.tools.list_ports.comports()
]
def main(ARGS):
model_name = "oliverguhr/wav2vec2-large-xlsr-53-german-cv9"
wave_buffer = BehaviorSubject(np.array([]))
wave2vec_asr = Wave2Vec2Inference(model_name)
wave_buffer.subscribe(
on_next=lambda x: asr_output_formatter(wave2vec_asr, x))
# Start audio with VAD
vad_audio = VADAudio(aggressiveness=ARGS.webRTC_aggressiveness,
device=ARGS.device,
input_rate=ARGS.rate)
print("Listening (ctrl-C to exit)...")
frames = vad_audio.vad_collector()
# load silero VAD
torchaudio.set_audio_backend("soundfile")
model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad',
model=ARGS.silaro_model_name,
force_reload=ARGS.reload,
onnx=True)
(get_speech_timestamps, save_audio, read_audio, VADIterator,
collect_chunks) = utils
# Stream from microphone to Wav2Vec 2.0 using VAD
print("audio length\tinference time\ttext")
spinner = None
if not ARGS.nospinner:
spinner = Halo(spinner='line')
wav_data = bytearray()
try:
for frame in frames:
if frame is not None:
if spinner:
spinner.start()
wav_data.extend(frame)
else:
if spinner:
spinner.stop()
#print("webRTC has detected a possible speech")
newsound = np.frombuffer(wav_data, np.int16)
audio_float32 = Int2FloatSimple(newsound)
time_stamps = get_speech_timestamps(audio_float32,
model,
sampling_rate=ARGS.rate)
if (len(time_stamps) > 0):
#print("silero VAD has detected a possible speech")
wave_buffer.on_next(audio_float32.numpy())
else:
print("VAD detected noise")
wav_data = bytearray()
except KeyboardInterrupt:
exit()
class Element:
def __init__(self, shape, dimensions, width, height):
self.shape = shape
self.dimensions = dimensions
self.width = width
self.height = height
self.position = BehaviorSubject((0,0))
self.id = uuid.uuid4()
def setPosition(self, p):
self.position.on_next(p)
class TimeAgoText(urwid.WidgetWrap):
def __init__(self, *args, **kwargs):
self.from_time = None
new_args = list(args)
if len(new_args) > 0:
self.from_time = new_args[0]
new_args = new_args[1:]
self.format = kwargs.pop('format', '{}')
self.from_time = kwargs.pop('from_time', self.from_time)
self.default_text = kwargs.pop('default_text', '')
self.widget = None
self.parent_widget = kwargs.pop('parent_widget', None)
self.time_ago_S = BehaviorSubject('<Unknown>')
self._time_ago_subscription = None
self.create(*new_args, **kwargs)
super().__init__(self.widget)
def _subscribe_to_time_ago(self):
def subscription_action(new_value):
time_ago_text = self.format.format(new_value)
self.widget.set_text(time_ago_text)
self._time_ago_subscription = self.time_ago_S.subscribe(
subscription_action)
return self
def _create_counter_task(self):
async def recursive_waiter():
text, time = _time_ago(self.from_time)
if text is None:
text = self.default_text
self.time_ago_S.on_next(text)
await asyncio.sleep(time)
if time is not None:
await recursive_waiter()
asyncio.ensure_future(recursive_waiter())
def create(self, *args, **kwargs):
self.widget = urwid.Text(self.default_text, *args, **kwargs)
self._subscribe_to_time_ago()
self._create_counter_task()
class Store:
def __init__(self, reducer: Dict, init_value: Dict):
self._reducer = reducer
#Default Setter
if "SET_VALUE" in self._reducer:
raise KeyError("Reducer name 'SET_VALUE' is reserved")
self._reducer["SET_VALUE"] = set_value_reducer
self._state = BehaviorSubject({})
self._state.on_next(init_value)
def dispatch(self, name: str, payload: Dict = None):
if name in self._reducer:
self._state.on_next(self._reducer[name](self.state, payload))
def get_reducers(self):
return {name: partial(self.dispatch, name) for name in self._reducer}
def select_compiled(self, comp, built_in=None, logger=None):
def wrapper(state):
built_in.update(state)
try:
return eval_compiled(comp, variables=built_in)
except:
# Could not evaluate, probably wrong binding
if logger:
logger.warn(f"Error could not evaluate binding")
logger.warn(format_exc(limit=2))
return None
return self.select([wrapper])
def select(self, selectors: List):
if len(selectors) == 1 and callable(selectors[0]):
return self._state.pipe(rxop.map(selectors[0]),
rxop.distinct_until_changed())
else:
return self._state.pipe(
rxop.map(lambda data: safe_get(data, selectors)),
rxop.distinct_until_changed())
def select_by_path(self, path):
return self.select(path.split("."))
def subscribe(self, subscriber):
return self._state.subscribe(subscriber)
@property
def state(self) -> Dict:
return self._state.value
class SubjectiveRadioButton(QRadioButton):
def __init__(self, *args):
super().__init__(*args)
self.subject = BehaviorSubject(self.isChecked())
self.toggled.connect(lambda: self.subject.on_next(self.isChecked()))
self.subject.subscribe(self.update_ui)
def update_ui(self, value):
self.setChecked(value)
class MainViewModel:
def __init__(self):
self.note_repository = NoteRepository()
# Create notes field as a behavior subject with note from the business logic as an initial value
# Your code here
self.behavior = BehaviorSubject(self.note_repository.get_all_notes())
def add_note(self, note: str):
# Add note and emit event with new date to the subject
# Your code here
self.note_repository.add_note(note)
self.behavior.on_next(self.note_repository.get_all_notes())
def clear_all(self):
# Clear all note and emit event with new data to the subject
# Your code here
self.note_repository.clear_all_notes()
self.behavior.on_next(self.note_repository.get_all_notes())
class SubjectiveTextEdit(QTextEdit):
def __init__(self, *args):
super().__init__(*args)
self.subject = BehaviorSubject(self.toPlainText())
self.textChanged.connect(
lambda: self.subject.on_next(self.toPlainText()))
self.subject.subscribe(self.update_ui)
def update_ui(self, value):
if self.toPlainText() != value:
self.setPlainText(value)
class TabViewModel:
"""Informação global da tab"""
def __init__(self, share_x, max_plots, name):
self.sharexBehavior = BehaviorSubject(share_x)
self.maxPlotsBehavior = BehaviorSubject(max_plots)
self.nameBehavior = BehaviorSubject(name)
def dispose(self):
self.sharexBehavior.dispose()
self.maxPlotsBehavior.dispose()
self.nameBehavior.dispose()
def _get_sharex_(self):
return self.sharexBehavior.value
def _set_sharex_(self, s):
self.sharexBehavior.on_next(s)
sharex = property(_get_sharex_, _set_sharex_)
def _get_maxPlots_(self):
return self.maxPlotsBehavior.value
def _set_maxPlots_(self, m):
self.maxPlotsBehavior.on_next(m)
maxPlots = property(_get_maxPlots_, _set_maxPlots_)
def _get_name_(self):
return self.nameBehavior.value
def _set_name_(self, value):
self.nameBehavior.on_next(value)
name = property(_get_name_, _set_name_)
class MQTTSource(Node, rx.Observable):
def __init__(self, client: mqtt_wrapper.MQTTClientWrapper, topic: Union[str, None], qos=0, options=None,
properties=None, config=None, name=""):
self.options = options
self.properties = properties
self.qos = qos
self.topic = topic
self.client = client
self.message_subject = subject.Subject()
self.broker_subscribed = False
self.subscription: rx.disposable.Disposable = None
self.can_subscribe = BehaviorSubject(False)
Node.__init__(self, name=name, config=config)
rx.Observable.__init__(self, subscribe=self.subscribe_func)
def subscribe_func(self, observer, scheduler=None) -> rx_t.Disposable:
if not self.broker_subscribed:
self.client.connected_subject.subscribe(self.configure_subscription)
logging.debug("subscribe to on_connect event")
subscription = self.message_subject.pipe(operators.map(self.transform)).subscribe(observer)
return subscription
def configure_subscription(self, x):
if x:
if self.topic:
self.subscribe_on_broker(self.topic)
self.can_subscribe.on_next(True)
def transform(self, x):
return x
def subscribe_on_broker(self, topic):
self.subscription = self.client.subscribe(topic, self.qos, self.options, self.properties)
self.client.message_callback_add(topic,
lambda client, userdata, message: self.message_subject.on_next(message))
def __del__(self):
if self.subscription:
self.subscription.dispose()
class PlotItemViewModel:
def __init__(self, strVariavel, strQuery, color, plotType):
self.strVariavelBehavior = BehaviorSubject(strVariavel)
self.strQueryBehavior = BehaviorSubject(strQuery)
self.colorBehavior = BehaviorSubject(color)
self.plotTypeBehavior = BehaviorSubject(plotType)
def dispose(self):
self.strVariavelBehavior.dispose()
self.strQueryBehavior.dispose()
self.colorBehavior.dispose()
self.plotTypeBehavior.dispose()
def _get_strVariavel_(self):
return self.strVariavelBehavior.value
def _set_strVariavel_(self, value):
if value != self.strVariavelBehavior.value:
self.strVariavelBehavior.on_next(value)
strVariavel = property(_get_strVariavel_, _set_strVariavel_)
def _get_strQuery_(self):
return self.strQueryBehavior.value
def _set_strQuery_(self, value):
if value != self.strQueryBehavior.value:
self.strQueryBehavior.on_next(value)
strQuery = property(_get_strQuery_, _set_strQuery_)
def _get_color_(self):
return self.colorBehavior.value
def _set_color_(self, value):
if value[0] != self.colorBehavior.value[0] or value[
1] != self.colorBehavior.value[1] or value[
2] != self.colorBehavior.value[2]:
self.colorBehavior.on_next(value)
color = property(_get_color_, _set_color_)
def _get_plotType_(self):
return self.plotTypeBehavior.value
def _set_plotType_(self, value):
if value != self.plotTypeBehavior.value:
self.plotTypeBehavior.on_next(value)
plotType = property(_get_plotType_, _set_plotType_)
class ComboBox(QComboBox, Subjective):
def __init__(self, *args, subject=None, **kwargs):
super().__init__(*args, **kwargs)
self.subject = subject
if self.subject is None:
self.subject = BehaviorSubject(self.currentData())
self.currentIndexChanged.connect(
lambda: self.subject.on_next(self.currentText()))
self.subject.subscribe(self.update_ui)
def update_ui(self, value):
if self.currentText() != value:
self.setCurrentText(value)
class PlotViewModel:
def __init__(self, xVariavel, xQuery, grid):
self.xVariavelBehavior = BehaviorSubject(xVariavel)
self.xQueryBehavior = BehaviorSubject(xQuery)
self.gridBehavior = BehaviorSubject(grid)
# Fazer parte dos itens
# self.plotItems = [ for pltItem in items]
# Itens
def dispose(self):
self.xVariavelBehavior.dispose()
self.xQueryBehavior.dispose()
self.gridBehavior.dispose()
def _get_xVariavel_(self):
return self.xVariavelBehavior.value
def _set_xVariavel_(self, value):
if value != self.xVariavelBehavior.value:
self.xVariavelBehavior.on_next(value)
xVariavel = property(_get_xVariavel_, _set_xVariavel_)
def _get_xQuery_(self):
return self.xQueryBehavior.value
def _set_xQuery_(self, value):
if value != self.xQueryBehavior.value:
self.xQueryBehavior.on_next(value)
xQuery = property(_get_xQuery_, _set_xQuery_)
def _get_grid_(self):
return self.gridBehavior.value
def _set_grid_(self, value):
if value != self.gridBehavior.value:
self.gridBehavior.on_next(value)
grid = property(_get_grid_, _set_grid_)
class MockLobbyServer:
def __init__(self):
self.game_msg = MockMessage()
self.player_msg = MockMessage()
self.login_msg = MockLoginMessage()
self.obs_connection_state = BehaviorSubject(
ConnectionState.DISCONNECTED)
@property
def connection_state(self):
return self.obs_connection_state.value
def connect(self):
QTimer.singleShot(0, self._set_connected)
def disconnect(self):
QTimer.singleShot(0, self._set_disconnected)
def _set_connected(self):
self.obs_connection_state.on_next(ConnectionState.CONNECTING)
self.obs_connection_state.on_next(ConnectionState.CONNECTED)
def _set_disconnected(self):
self.obs_connection_state.on_next(ConnectionState.DISCONNECTED)
class Client(threading.Thread):
connection: websockets.WebSocketClientProtocol
def __init__(self, options: ContextOptions):
self.connected = BehaviorSubject(False)
self.options: ContextOptions = options
self.loop = asyncio.new_event_loop()
self.host = os.environ.get('DEEPKIT_HOST', '127.0.0.1')
self.port = int(os.environ.get('DEEPKIT_PORT', '8960'))
self.token = os.environ.get('DEEPKIT_JOB_ACCESSTOKEN', None)
self.job_id = os.environ.get('DEEPKIT_JOB_ID', None)
self.message_id = 0
self.account = 'localhost'
self.callbacks: Dict[int, asyncio.Future] = {}
self.subscriber: Dict[int, any] = {}
self.stopping = False
self.queue = []
self.controllers = {}
self.patches = {}
self.offline = False
self.connections = 0
self.lock = threading.Lock()
threading.Thread.__init__(self)
self.daemon = True
self.loop = asyncio.new_event_loop()
self.start()
def run(self):
self.connecting = self.loop.create_future()
self.loop.create_task(self._connect())
self.loop.run_forever()
def shutdown(self):
if self.offline: return
promise = asyncio.run_coroutine_threadsafe(self.stop_and_sync(),
self.loop)
promise.result()
if not self.connection.closed:
raise Exception('Connection still active')
self.loop.stop()
async def stop_and_sync(self):
self.stopping = True
# done = 150, //when all tasks are done
# aborted = 200, //when at least one task aborted
# failed = 250, //when at least one task failed
# crashed = 300, //when at least one task crashed
self.patches['status'] = 150
self.patches['ended'] = datetime.utcnow().isoformat()
self.patches['tasks.main.ended'] = datetime.utcnow().isoformat()
# done = 500,
# aborted = 550,
# failed = 600,
# crashed = 650,
self.patches['tasks.main.status'] = 500
self.patches['tasks.main.instances.0.ended'] = datetime.utcnow(
).isoformat()
# done = 500,
# aborted = 550,
# failed = 600,
# crashed = 650,
self.patches['tasks.main.instances.0.status'] = 500
if hasattr(sys, 'last_value'):
if isinstance(sys.last_value, KeyboardInterrupt):
self.patches['status'] = 200
self.patches['tasks.main.status'] = 550
self.patches['tasks.main.instances.0.status'] = 550
else:
self.patches['status'] = 300
self.patches['tasks.main.status'] = 650
self.patches['tasks.main.instances.0.status'] = 650
while len(self.patches) > 0 or len(self.queue) > 0:
await asyncio.sleep(0.15)
await self.connection.close()
async def register_controller(self, name: str, controller):
self.controllers[name] = controller
async def subscriber(message, done):
if message['type'] == 'error':
done()
del self.controllers[name]
raise Exception('Register controller error: ' +
message['error'])
if message['type'] == 'ack':
pass
if message['type'] == 'peerController/message':
data = message['data']
if not hasattr(controller, data['action']):
error = f"Requested action {message['action']} not available in {name}"
print(error, file=sys.stderr)
await self._message(
{
'name': 'peerController/message',
'controllerName': name,
'clientId': message['clientId'],
'data': {
'type': 'error',
'id': data['id'],
'stack': None,
'entityName': '@error:default',
'error': error
}
},
no_response=True)
if data['name'] == 'actionTypes':
parameters = []
i = 0
for arg in inspect.getfullargspec(
getattr(controller, data['action'])).args:
parameters.append({
'type': 'any',
'name': '#' + str(i)
})
i += 1
await self._message(
{
'name': 'peerController/message',
'controllerName': name,
'clientId': message['clientId'],
'data': {
'type': 'actionTypes/result',
'id': data['id'],
'parameters': parameters,
'returnType': {
'type': 'any',
'name': 'result'
}
}
},
no_response=True)
if data['name'] == 'action':
try:
res = getattr(controller,
data['action'])(*data['args'])
await self._message(
{
'name': 'peerController/message',
'controllerName': name,
'clientId': message['clientId'],
'data': {
'type': 'next/json',
'id': data['id'],
'encoding': {
'name': 'r',
'type': 'any'
},
'next': res,
}
},
no_response=True)
except Exception as e:
await self._message(
{
'name': 'peerController/message',
'controllerName': name,
'clientId': message['clientId'],
'data': {
'type': 'error',
'id': data['id'],
'stack': None,
'entityName': '@error:default',
'error': str(e)
}
},
no_response=True)
await self._subscribe(
{
'name': 'peerController/register',
'controllerName': name,
}, subscriber)
class Controller:
def __init__(self, client):
self.client = client
def stop(self):
self.client._message({
'name': 'peerController/unregister',
'controllerName': name,
})
return Controller(self)
async def _action(self,
controller: str,
action: str,
args: List,
lock=True,
allow_in_shutdown=False):
if lock: await self.connecting
if self.offline: return
if self.stopping and not allow_in_shutdown:
raise Exception('In shutdown: actions disallowed')
if not controller: raise Exception('No controller given')
if not action: raise Exception('No action given')
res = await self._message(
{
'name': 'action',
'controller': controller,
'action': action,
'args': args,
'timeout': 60
},
lock=lock)
if res['type'] == 'next/json':
return res['next'] if 'next' in res else None
if res['type'] == 'error':
print(res, file=sys.stderr)
raise ApiError('API Error: ' + str(res['error']))
raise ApiError(f"Invalid action type '{res['type']}'. Not implemented")
def job_action(self, action: str, args: List):
return asyncio.run_coroutine_threadsafe(
self._action('job', action, args), self.loop)
async def _subscribe(self, message, subscriber):
await self.connecting
self.message_id += 1
message['id'] = self.message_id
message_id = self.message_id
def on_done():
del self.subscriber[message_id]
async def on_incoming_message(incoming_message):
await subscriber(incoming_message, on_done)
self.subscriber[self.message_id] = on_incoming_message
self.queue.append(message)
async def _message(self, message, lock=True, no_response=False):
if lock: await self.connecting
self.message_id += 1
message['id'] = self.message_id
if not no_response:
self.callbacks[self.message_id] = self.loop.create_future()
self.queue.append(message)
if no_response:
return
return await self.callbacks[self.message_id]
def patch(self, path: str, value: any):
if self.offline: return
if self.stopping: return
self.patches[path] = value
async def send_messages(self, connection):
while not connection.closed:
try:
q = self.queue[:]
for m in q:
await connection.send(json.dumps(m))
self.queue.remove(m)
except Exception as e:
print("Failed sending, exit send_messages")
raise e
if len(self.patches) > 0:
# we have to send first all messages/actions out
# before sending patches, as most of the time
# patches are based on previously created entities,
# so we need to make sure those entities are created
# first before sending any patches.
try:
send = self.patches.copy()
await connection.send(
json.dumps({
'name': 'action',
'controller': 'job',
'action': 'patchJob',
'args': [send],
'timeout': 60
}))
for i in send.keys():
if self.patches[i] == send[i]:
del self.patches[i]
except websockets.exceptions.ConnectionClosed:
return
except ApiError:
print("Patching failed. Syncing job data disabled.",
file=sys.stderr)
return
await asyncio.sleep(0.2)
async def handle_messages(self, connection):
while not connection.closed:
try:
res = json.loads(await connection.recv())
except websockets.exceptions.ConnectionClosedError:
# we need reconnect
break
except websockets.exceptions.ConnectionClosedOK:
# we closed on purpose, so no reconnect necessary
return
if res and 'id' in res:
if res['id'] in self.subscriber:
await self.subscriber[res['id']](res)
if res['id'] in self.callbacks:
self.callbacks[res['id']].set_result(res)
del self.callbacks[res['id']]
if not self.stopping:
print("Deepkit: lost connection. reconnect ...")
self.connecting = self.loop.create_future()
self.connected.on_next(False)
self.loop.create_task(self._connect())
async def _connect_job(self, host: str, port: int, id: str, token: str):
try:
self.connection = await websockets.connect(f"ws://{host}:{port}")
except Exception:
# try again later
await asyncio.sleep(1)
self.loop.create_task(self._connect())
return
self.loop.create_task(self.handle_messages(self.connection))
self.loop.create_task(self.send_messages(self.connection))
res = await self._message(
{
'name': 'authenticate',
'token': {
'id': 'job',
'token': token,
'job': id
}
},
lock=False)
if not res['result'] or res['result'] is not True:
raise Exception('Job token invalid')
self.connecting.set_result(True)
if self.connections > 0:
print("Deepkit: Reconnected.")
self.connected.on_next(True)
self.connections += 1
async def _connect(self):
# we want to restart with a empty queue, so authentication happens always first
queue_copy = self.queue[:]
self.queue = []
if self.token:
await self._connect_job(self.host, self.port, self.job_id,
self.token)
else:
config = get_home_config()
link: Optional[FolderLink] = None
if self.options.account:
account_config = config.get_account_for_name(
self.options.account)
else:
link = config.get_folder_link_of_directory(sys.path[0])
account_config = config.get_account_for_id(link.accountId)
self.host = account_config.host
self.port = account_config.port
ws = 'wss' if account_config.ssl else 'ws'
try:
self.connection = await websockets.connect(
f"{ws}://{self.host}:{self.port}")
except Exception as e:
self.offline = True
print(
f"Deepkit: App not started or server not reachable. Monitoring disabled. {e}"
)
self.connecting.set_result(False)
return
self.loop.create_task(self.handle_messages(self.connection))
self.loop.create_task(self.send_messages(self.connection))
res = await self._message(
{
'name': 'authenticate',
'token': {
'id': 'user',
'token': account_config.token
}
},
lock=False)
if not res['result']:
raise Exception('Login invalid')
if link:
projectId = link.projectId
else:
if not self.options.project:
raise Exception(
'No project defined. Please use ContextOptions(project="project-name") '
'to specify which project to use.')
project = await self._action('app',
'getProjectForPublicName',
[self.options.project],
lock=False)
if not project:
raise Exception(
f'No project found for name {self.options.project}. '
f'Do you use the correct account? (used {account_config.name})'
)
projectId = project['id']
job = await self._action('app',
'createJob', [projectId],
lock=False)
deepkit.globals.loaded_job = job
self.token = await self._action('app',
'getJobAccessToken', [job['id']],
lock=False)
self.job_id = job['id']
# todo, implement re-authentication, so we don't have to drop the active connection
await self.connection.close()
await self._connect_job(self.host, self.port, self.job_id,
self.token)
self.queue = queue_copy + self.queue
class BlenderContext(Context):
window_size: RV[Dimension] = rv.new_view()
batch: RV[GPUBatch] = window_size.map(lambda c, s: c.create_batch(s))
buffer: RV[GPUBatch] = window_size.map(lambda c, s: c.create_batch(s))
def __init__(self,
toolkit: BlenderToolkit,
look_and_feel: Optional[LookAndFeel] = None,
font_options: Optional[FontOptions] = None,
window_manager: Optional[WindowManager] = None,
error_handler: Optional[ErrorHandler] = None) -> None:
super().__init__(toolkit, look_and_feel, font_options, window_manager,
error_handler)
resolution = Dimension(bge.render.getWindowWidth(),
bge.render.getWindowHeight())
self._resolution = BehaviorSubject(resolution)
# noinspection PyTypeChecker
self.window_size = self._resolution.pipe(ops.distinct_until_changed())
self._shader = cast(GPUShader, gpu.shader.from_builtin("2D_IMAGE"))
if use_viewport_render:
# noinspection PyArgumentList
self._draw_handler = SpaceView3D.draw_handler_add(
self.process, (), "WINDOW", "POST_PIXEL")
else:
self._draw_handler = None
bge.logic.getCurrentScene().post_draw.append(self.process)
# noinspection PyTypeChecker
self._texture = Buffer(bgl.GL_INT, 1)
bgl.glGenTextures(1, self.texture)
@property
def shader(self) -> GPUShader:
return self._shader
@property
def texture(self) -> Buffer:
return self._texture
def create_batch(self, size: Dimension) -> GPUBatch:
if size is None:
raise ValueError("Argument 'size' is required.")
points = Bounds(0, 0, size.width, size.height).points
vertices = tuple(map(lambda p: p.tuple, map(self.translate, points)))
coords = ((0, 0), (1, 0), (1, 1), (0, 1))
indices = {"pos": vertices, "texCoord": coords}
return batch_for_shader(self.shader, "TRI_FAN", indices)
def translate(self, point: Point) -> Point:
if point is None:
raise ValueError("Argument 'point' is required.")
return point.copy(y=self.window_size.height - point.y)
# noinspection PyTypeChecker
def process_draw(self) -> None:
width = bge.render.getWindowWidth()
height = bge.render.getWindowHeight()
self._resolution.on_next(Dimension(width, height))
super().process_draw()
data = self.surface.get_data()
source = bgl.Buffer(bgl.GL_BYTE, width * height * 4, data)
bgl.glEnable(bgl.GL_BLEND)
bgl.glActiveTexture(bgl.GL_TEXTURE0)
# noinspection PyUnresolvedReferences
bgl.glBindTexture(bgl.GL_TEXTURE_2D, self.texture[0])
bgl.glTexImage2D(bgl.GL_TEXTURE_2D, 0, bgl.GL_SRGB_ALPHA, width,
height, 0, bgl.GL_BGRA, bgl.GL_UNSIGNED_BYTE, source)
bgl.glTexParameteri(bgl.GL_TEXTURE_2D, bgl.GL_TEXTURE_MIN_FILTER,
bgl.GL_NEAREST)
bgl.glTexParameteri(bgl.GL_TEXTURE_2D, bgl.GL_TEXTURE_MAG_FILTER,
bgl.GL_NEAREST)
self.shader.bind()
self.shader.uniform_int("image", 0)
self.batch.draw(self.shader)
bgl.glDeleteBuffers(1, source)
def dispose(self) -> None:
if self._draw_handler:
# noinspection PyArgumentList
SpaceView3D.draw_handler_remove(self._draw_handler, "WINDOW")
else:
bge.logic.getCurrentScene().post_draw.remove(self.process)
bgl.glDeleteTextures(1, self.texture)
# noinspection PyTypeChecker
bgl.glDeleteBuffers(1, self.texture)
super().dispose()
class DevRantService():
def __init__(self, *args, **kwargs):
self.base_url = "https://devrant.com/api"
self.base_params = {
'app': 3,
}
self.rants = BehaviorSubject([])
self.error = Subject()
self.me = BehaviorSubject(None)
self.auth_token = BehaviorSubject(None)
self.is_logged = BehaviorSubject(False)
self._subscribe_to_auth_token()
self._load_cache()
def __del__(self):
self.rants.dispose()
self.error.dispose()
self.me.dispose()
self.auth_token.dispose()
self.is_logged.dispose()
def _get_params(self, **kwargs):
params = dict(self.base_params, **kwargs)
return urllib.parse.urlencode(params)
def _load_cache(self, filename='.cache.json'):
try:
fh = open(filename, 'r')
except FileNotFoundError:
self._write_cache(filename)
self._load_cache(filename)
else:
try:
cache_data = json.load(fh)
except json.decoder.JSONDecodeError:
pass
else:
cached_auth_token = cache_data.get('auth_token')
if cached_auth_token is not None:
cached_auth_token = AuthToken(**cached_auth_token)
self.auth_token.on_next(cached_auth_token)
fh.close()
def _write_cache(self, filename='.cache.json'):
cache_data = {}
try:
fh = open(filename, 'r')
cache_data = json.load(fh)
except FileNotFoundError:
pass
except json.JSONDecodeError:
pass
else:
fh.close()
fh = open(filename, 'w')
if self.auth_token.value is not None:
cache_data['auth_token'] = self.auth_token.value.__dict__()
json.dump(cache_data, fh)
fh.close()
def _delete_cache(self, filename='.cache.json'):
fh = open(filename, 'w')
json.dump({}, fh)
fh.close()
def _subscribe_to_auth_token(self):
def action(value):
# change is_logged value
if self.is_logged.value and value.user_id is None:
self.is_logged.on_next(False)
elif not self.is_logged.value and value and value.user_id:
self.is_logged.on_next(True)
# save new auth_token
self._write_cache()
self._auth_token_subscription = self.auth_token.subscribe(action)
return self
async def get_rants(self, limit=10):
param_url = self._get_params(sort='recent',
limit=limit,
skip=len(self.rants.value))
response = requests.get(self.base_url + '/devrant/rants' + '?' +
param_url)
if response.status_code == 200:
new_rants = json.loads(response.text)['rants']
new_rants = [Rant(rant) for rant in new_rants]
all_rants = list(
merge(self.rants.value,
new_rants,
key=lambda x: x.id,
reverse=True))
self.rants.on_next(all_rants)
else:
self.error.on_next({
'code': 1,
'message': 'Unexpected status code',
'response': response
})
return self
async def get_new_rants(self, skip=0, limit=10):
if len(self.rants.value) == 0:
return await self.get_rants()
newest_actual_rant = self.rants.value[0]
param_url = self._get_params(sort='recent', limit=limit, skip=skip)
response = requests.get(self.base_url + '/devrant/rants' + '?' +
param_url)
if response.status_code == 200:
new_rants = json.loads(response.text)['rants']
new_rants = [Rant(rant) for rant in new_rants]
new_rants.sort(key=lambda x: x.id, reverse=True)
if new_rants[-1].id > newest_actual_rant.id:
await self.get_new_rants(skip + limit, limit)
newest_actual_rant = self.rants.value[0]
new_rants = [
rant for rant in new_rants if rant.id > newest_actual_rant.id
]
all_rants = list(
merge(new_rants,
self.rants.value,
key=lambda x: x.id,
reverse=True))
self.rants.on_next(all_rants)
else:
self.error.on_next({
'code': 1,
'message': 'Unexpected status code',
'response': response
})
return self
async def post_rant(self, draft_rant):
headers = {
"Host": "devrant.com",
"Connection": "keep-alive",
"Accept": "application/json",
"Origin": "https://devrant.com",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Referer": "https://devrant.com/feed",
}
form_data = {
**self.base_params, 'rant': draft_rant.text,
'token_id': self.auth_token.value.id,
'token_key': self.auth_token.value.key,
'user_id': self.auth_token.value.user_id,
'tags': ', '.join(draft_rant.tags),
'type': 1
}
draft_rant.response = requests.post(self.base_url + '/devrant/rants',
headers=headers,
data=form_data)
draft_rant.state.on_next(DraftState.Sent)
if draft_rant.response.status_code == 200:
success = json.loads(draft_rant.response.text).get('success')
if success:
await self.get_new_rants()
draft_rant.state.on_next(DraftState.Published)
else:
draft_rant.state.on_next(DraftState.Rejected)
self.error.on_next({
'code': 2,
'message': 'Posted rant error',
'response': draft_rant.response
})
elif draft_rant.response.status_code == 400:
draft_rant.state.on_next(DraftState.Rejected)
self.error.on_next({
'code': 2,
'message': 'Posted rant error',
'response': draft_rant.response
})
else:
draft_rant.state.on_next(DraftState.Rejected)
self.error.on_next({
'code': 1,
'message': 'Unexpected status code',
'response': draft_rant.response
})
async def login(self, username, password):
headers = {
"Host": "devrant.com",
"Connection": "keep-alive",
"Accept": "application/json",
"Origin": "https://devrant.com",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Referer": "https://devrant.com/",
}
response = requests.post(self.base_url + '/users/auth-token',
headers=headers,
data={
**self.base_params,
"username": username,
"password": password,
})
if response.status_code == 200:
data = response.json()
if data.get('success'):
auth_token = AuthToken(**data.get('auth_token'))
self.auth_token.on_next(auth_token)
# await self.get_about_me()
else:
self.error.on_next({'message': data.get('error')})
def logout(self):
self._delete_cache()
self.auth_token.on_next(AuthToken())
async def get_about_me(self):
param_url = self._get_params(token_id=self.auth_token.value.id,
token_key=self.auth_token.value.key)
response = requests.get(
self.base_url + '/users/{}'.format(self.auth_token.value.user_id) +
'?' + param_url)
return response
class SubjectiveComboBox(QComboBox):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.subject = BehaviorSubject(self.currentData())
self.currentIndexChanged.connect(
lambda: self.subject.on_next(self.currentText()))
class GenericRantList(urwid.WidgetWrap):
def __init__(self, rants_subscriptable):
self.widget = None
self.rants_subscriptable = rants_subscriptable
self.rants = []
self.update = BehaviorSubject(self)
self._subscriptions = []
self.create()
super().__init__(self.widget)
def __del__(self):
self.update.dispose()
for subscription in self._subscritpions:
subscription.unsubscribe()
def create_list_box(self):
elements = []
for i, rant in enumerate(self.rants):
if i > 0:
elements.append(urwid.Divider(u'\u2500'))
element = RantListElementWidget(rant)
elements.append(element)
list_box = urwid.ListBox(urwid.SimpleListWalker(elements))
return list_box
def create(self):
self.widget = urwid.Frame(self.create_list_box())
self._subscribe_rant_list()
def _subscribe_rant_list(self):
def action(new_value):
self.rants = new_value
simple_list_walker = self.widget.contents['body'][0].body
i = 0
j = 0
# update existent
while i < len(simple_list_walker) and j < len(new_value):
element = simple_list_walker[i]
rant = new_value[j]
if type(element) is not RantListElementWidget:
i += 1
else:
element.update_rant(rant)
i += 1
j += 1
# append new ones
while j < len(new_value):
rant = new_value[j]
if i > 0:
simple_list_walker.append(urwid.Divider(u'\u2500'))
i += 1
simple_list_walker.contents.append(RantListElementWidget(rant))
i += 1
j += 1
# delete excedent
simple_list_walker.contents[:i]
self.update.on_next(self)
self._subscriptions.append(self.rants_subscriptable.subscribe(action))
class Client(threading.Thread):
connection: websockets.WebSocketClientProtocol
def __init__(self,
project: Optional[str] = None,
account: Optional[str] = None,
try_pick_up=False,
parent_experiment=None,
silent=False):
self.connected = BehaviorSubject(False)
self.project = project
self.account = account
self.parent_experiment = parent_experiment
self.silent = silent
self.host = os.environ.get('DEEPKIT_HOST', '127.0.0.1')
self.socket_path = os.environ.get('DEEPKIT_SOCKET', None)
self.ssl = os.environ.get('DEEPKIT_SSL', '0') is '1'
self.port = int(os.environ.get('DEEPKIT_PORT', '8960'))
self.job_token = None
self.job_id = None
if try_pick_up:
# is set by Deepkit cli
self.job_token = os.environ.get('DEEPKIT_JOB_ACCESSTOKEN', None)
self.job_id = os.environ.get('DEEPKIT_JOB_ID', None)
# is set by deepkit.login()
self.token = os.environ.get('DEEPKIT_ACCESSTOKEN', None)
self.result_status = None
self.message_id = 0
self.callbacks: Dict[int, asyncio.Future] = {}
self.subscriber: Dict[int, any] = {}
self.stopping = False
self.queue = []
self.controllers = {}
self.patches = {}
self.offline = False
self.connections = 0
self.lock = threading.Lock()
threading.Thread.__init__(self)
self.daemon = True
self.loop = asyncio.new_event_loop()
self.start()
def is_connected(self):
return self.connected.value
def run(self):
self.connecting = self.loop.create_future()
self.loop.run_forever()
def connect(self):
asyncio.run_coroutine_threadsafe(self._connect(), self.loop)
def connect_anon(self):
asyncio.run_coroutine_threadsafe(self._connect_anon(),
self.loop).result()
def shutdown(self):
if self.offline: return
promise = asyncio.run_coroutine_threadsafe(self.stop_and_sync(),
self.loop)
promise.result()
if not self.connection.closed:
raise Exception('Connection still active')
self.loop.stop()
async def stop_and_sync(self):
self.stopping = True
if deepkit.utils.in_self_execution() or self.result_status:
# only when we are in self execution do we set status, time stamps etc
# otherwise the CLI is doing that and the server. Or when
# the experiment set result_state explicitly.
# done = 150, //when all tasks are done
# aborted = 200, //when at least one task aborted
# failed = 250, //when at least one task failed
# crashed = 300, //when at least one task crashed
self.patches['status'] = 150
self.patches['ended'] = datetime.datetime.utcnow()
self.patches['tasks.main.ended'] = datetime.datetime.utcnow()
# done = 500,
# aborted = 550,
# failed = 600,
# crashed = 650,
self.patches['tasks.main.status'] = 500
self.patches[
'tasks.main.instances.0.ended'] = datetime.datetime.utcnow()
# done = 500,
# aborted = 550,
# failed = 600,
# crashed = 650,
self.patches['tasks.main.instances.0.status'] = 500
if hasattr(sys, 'last_value'):
if isinstance(sys.last_value, KeyboardInterrupt):
self.patches['status'] = 200
self.patches['tasks.main.status'] = 550
self.patches['tasks.main.instances.0.status'] = 550
else:
self.patches['status'] = 300
self.patches['tasks.main.status'] = 650
self.patches['tasks.main.instances.0.status'] = 650
if self.result_status:
self.patches['status'] = self.result_status.value
while len(self.patches) > 0 or len(self.queue) > 0:
await asyncio.sleep(0.15)
await self.connection.close()
def register_controller(self, name: str, controller):
return asyncio.run_coroutine_threadsafe(
self._register_controller(name, controller), self.loop)
async def _register_controller(self, name: str, controller):
self.controllers[name] = controller
async def handle_peer_message(message, done):
if message['type'] == 'error':
done()
del self.controllers[name]
raise Exception('Register controller error: ' +
message['error'])
if message['type'] == 'ack':
pass
if message['type'] == 'peerController/message':
data = message['data']
if not hasattr(controller, data['action']):
error = f"Requested action {message['action']} not available in {name}"
print(error, file=sys.stderr)
await self._message(
{
'name': 'peerController/message',
'controllerName': name,
'clientId': message['clientId'],
'data': {
'type': 'error',
'id': data['id'],
'stack': None,
'entityName': '@error:default',
'error': error
}
},
no_response=True)
if data['name'] == 'actionTypes':
parameters = []
i = 0
for arg in inspect.getfullargspec(
getattr(controller, data['action'])).args:
parameters.append({
'type': 'any',
'name': '#' + str(i)
})
i += 1
await self._message(
{
'name': 'peerController/message',
'controllerName': name,
'clientId': message['clientId'],
'data': {
'type': 'actionTypes/result',
'id': data['id'],
'parameters': parameters,
'returnType': {
'type': 'any',
'name': 'result'
}
}
},
no_response=True)
if data['name'] == 'action':
try:
res = await getattr(controller,
data['action'])(*data['args'])
await self._message(
{
'name': 'peerController/message',
'controllerName': name,
'clientId': message['clientId'],
'data': {
'type': 'next/json',
'id': data['id'],
'encoding': {
'name': 'r',
'type': 'any'
},
'next': res,
}
},
no_response=True)
except Exception as e:
await self._message(
{
'name': 'peerController/message',
'controllerName': name,
'clientId': message['clientId'],
'data': {
'type': 'error',
'id': data['id'],
'stack': None,
'entityName': '@error:default',
'error': str(e)
}
},
no_response=True)
def subscriber(message, on_done):
self.loop.create_task(handle_peer_message(message, on_done))
await self._subscribe(
{
'name': 'peerController/register',
'controllerName': name,
}, subscriber)
class Controller:
def __init__(self, client):
self.client = client
def stop(self):
self.client._message(
{
'name': 'peerController/unregister',
'controllerName': name,
},
no_response=True)
return Controller(self)
async def _action(self,
controller: str,
action: str,
args=None,
lock=True,
allow_in_shutdown=False):
if args is None:
args = []
if lock: await self.connecting
if self.offline: return
if self.stopping and not allow_in_shutdown:
raise Exception('In shutdown: actions disallowed')
if not controller: raise Exception('No controller given')
if not action: raise Exception('No action given')
# print('> action', action, threading.current_thread().name)
res = await self._message(
{
'name': 'action',
'controller': controller,
'action': action,
'args': args,
'timeout': 60
},
lock=lock)
# print('< action', action)
if res['type'] == 'next/json':
return res['next'] if 'next' in res else None
if res['type'] == 'error':
print(res, file=sys.stderr)
raise ApiError('API Error: ' + str(res['error']))
raise ApiError(f"Invalid action type '{res['type']}'. Not implemented")
def app_action_threadsafe(self, action: str, args=None) -> Future:
if args is None: args = []
return asyncio.run_coroutine_threadsafe(
self._action('app', action, args), self.loop)
async def job_action(self, action: str, args=None):
return await self._action('job', action, args)
def job_action_threadsafe(self, action: str, args=None) -> Future:
"""
This method is non-blocking and every try to block-wait for an answers means
script execution stops when connection is broken (offline training entirely impossible).
So, we just schedule the call and return a Future, which the user can subscribe to.
"""
if args is None: args = []
return asyncio.run_coroutine_threadsafe(
self._action('job', action, args), self.loop)
async def _subscribe(self, message, subscriber):
await self.connecting
self.message_id += 1
message['id'] = self.message_id
message_id = self.message_id
def on_done():
del self.subscriber[message_id]
def on_incoming_message(incoming_message):
subscriber(incoming_message, on_done)
self.subscriber[self.message_id] = on_incoming_message
self.queue.append(message)
def _create_message(self,
message: dict,
lock=True,
no_response=False) -> dict:
self.message_id += 1
message['id'] = self.message_id
if not no_response:
self.callbacks[self.message_id] = self.loop.create_future()
return message
async def _message(self, message, lock=True, no_response=False):
if lock: await self.connecting
message = self._create_message(message, no_response=no_response)
self.queue.append(message)
if no_response:
return
return await self.callbacks[self.message_id]
def patch(self, path: str, value: any):
if self.offline: return
if self.stopping: return
self.patches[path] = value
async def send_messages(self, connection):
while not connection.closed:
try:
q = self.queue[:]
for m in q:
try:
j = json.dumps(m, default=json_converter)
except TypeError as e:
print('Could not send message since JSON error',
e,
m,
file=sys.stderr)
continue
await connection.send(j)
self.queue.remove(m)
except Exception as e:
print("Failed sending, exit send_messages", file=sys.stderr)
raise e
if len(self.patches) > 0:
# we have to send first all messages/actions out
# before sending patches, as most of the time
# patches are based on previously created entities,
# so we need to make sure those entities are created
# first before sending any patches.
# print('patches', self.patches)
try:
send = self.patches.copy()
await connection.send(
json.dumps(
{
'name': 'action',
'controller': 'job',
'action': 'patchJob',
'args': [send],
'timeout': 60
},
default=json_converter))
for i in send.keys():
if self.patches[i] == send[i]:
del self.patches[i]
except websockets.exceptions.ConnectionClosed:
return
except ApiError:
print("Patching failed. Syncing job data disabled.",
file=sys.stderr)
return
await asyncio.sleep(0.5)
async def handle_messages(self, connection):
while not connection.closed:
try:
res = json.loads(await connection.recv())
except websockets.exceptions.ConnectionClosedError:
# we need reconnect
break
except websockets.exceptions.ConnectionClosedOK:
# we closed on purpose, so no reconnect necessary
return
if res and 'id' in res:
if res['id'] in self.subscriber:
self.subscriber[res['id']](res)
if res['id'] in self.callbacks:
self.callbacks[res['id']].set_result(res)
del self.callbacks[res['id']]
if not self.stopping:
self.log("Deepkit: lost connection. reconnect ...")
self.connecting = self.loop.create_future()
self.connected.on_next(False)
self.loop.create_task(self._connect())
async def _connected(self, id: str, token: str):
try:
if self.socket_path:
self.connection = await websockets.unix_connect(
self.socket_path)
else:
ws = 'wss' if self.ssl else 'ws'
url = f"{ws}://{self.host}:{self.port}"
self.connection = await websockets.connect(url)
except Exception as e:
# try again later
self.log('Unable to connect', e)
await asyncio.sleep(1)
self.loop.create_task(self._connect())
return
self.loop.create_task(self.handle_messages(self.connection))
# we don't use send_messages() since this would send all queue/patches
# which would lead to permission issues when we're not first authenticated
if token:
message = self._create_message(
{
'name': 'authenticate',
'token': {
'id': 'job',
'token': token,
'job': id
}
},
lock=False)
await self.connection.send(
json.dumps(message, default=json_converter))
res = await self.callbacks[message['id']]
if not res['result'] or res['result'] is not True:
raise Exception('Job token invalid')
self.loop.create_task(self.send_messages(self.connection))
self.connecting.set_result(True)
if self.connections > 0:
self.log("Deepkit: Reconnected.")
self.connected.on_next(True)
self.connections += 1
async def _connect_anon(self):
ws = 'wss' if self.ssl else 'ws'
url = f"{ws}://{self.host}:{self.port}"
self.connection = await websockets.connect(url)
self.loop.create_task(self.handle_messages(self.connection))
self.loop.create_task(self.send_messages(self.connection))
self.connecting.set_result(True)
self.connected.on_next(True)
self.connections += 1
async def _connect(self):
# we want to restart with a empty queue, so authentication happens always first
queue_copy = self.queue[:]
self.queue = []
if self.job_token:
await self._connected(self.job_id, self.job_token)
return
try:
link: Optional[FolderLink] = None
user_token = self.token
account_name = 'none'
if not user_token:
config = get_home_config()
# when no user_token is given (via deepkit.login() for example)
# we need to find the host, port, token from the user config in ~/.deepkit/config
if not self.account and not self.project:
# find both, start with
link = config.get_folder_link_of_directory(sys.path[0])
account_config = config.get_account_for_id(link.accountId)
elif self.account and not self.project:
account_config = config.get_account_for_name(self.account)
else:
# default to first account configured
account_config = config.get_first_account()
account_name = account_config.name
self.host = account_config.host
self.port = account_config.port
self.ssl = account_config.ssl
user_token = account_config.token
ws = 'wss' if self.ssl else 'ws'
try:
url = f"{ws}://{self.host}:{self.port}"
self.connection = await websockets.connect(url)
except Exception as e:
self.offline = True
print(
f"Deepkit: App not started or server not reachable. Monitoring disabled. {e}",
file=sys.stderr)
self.connecting.set_result(False)
return
self.loop.create_task(self.handle_messages(self.connection))
self.loop.create_task(self.send_messages(self.connection))
res = await self._message(
{
'name': 'authenticate',
'token': {
'id': 'user',
'token': user_token
}
},
lock=False)
if not res['result']:
raise Exception('Login invalid')
project_name = ''
if link:
project_name = link.name
projectId = link.projectId
else:
if not self.project:
raise Exception(
'No project defined. Please use project="project-name" '
'to specify which project to use.')
project = await self._action('app',
'getProjectForPublicName',
[self.project],
lock=False)
if not project:
raise Exception(
f'No project found for name {self.project}. Make sure it exists before using it. '
f'Do you use the correct account? (used {account_name})'
)
project_name = project['name']
projectId = project['id']
job = await self._action('app',
'createJob',
[projectId, self.parent_experiment],
lock=False)
prefix = "Sub experiment" if self.parent_experiment else "Experiment"
self.log(
f"{prefix} #{job['number']} created in project {project_name} using account {account_name}"
)
deepkit.globals.loaded_job_config = job['config']['config']
self.job_token = await self._action('app',
'getJobAccessToken',
[job['id']],
lock=False)
self.job_id = job['id']
# todo, implement re-authentication, so we don't have to drop the active connection
await self.connection.close()
await self._connected(self.job_id, self.job_token)
except Exception as e:
self.connecting.set_exception(e)
self.queue = queue_copy + self.queue
def log(self, *message: str):
if not self.silent: print(*message)
class Client(threading.Thread):
connection: websockets.WebSocketClientProtocol
def __init__(self, config_path: str):
self.connected = BehaviorSubject(False)
self.config_path = config_path
self.loop = asyncio.new_event_loop()
self.host = os.environ.get('DEEPKIT_HOST', '127.0.0.1')
self.port = int(os.environ.get('DEEPKIT_PORT', '8960'))
self.token = os.environ.get('DEEPKIT_JOB_ACCESSTOKEN', None)
self.job_id = os.environ.get('DEEPKIT_JOB_ID', None)
self.message_id = 0
self.account = 'localhost'
self.callbacks: Dict[int, asyncio.Future] = {}
self.subscriber: Dict[int, any] = {}
self.stopping = False
self.queue = []
self.controllers = {}
self.patches = {}
self.offline = False
self.connections = 0
self.lock = threading.Lock()
threading.Thread.__init__(self)
self.daemon = True
self.loop = asyncio.new_event_loop()
self.start()
def run(self):
self.connecting = self.loop.create_future()
self.loop.create_task(self._connect())
self.loop.run_forever()
def shutdown(self):
if self.offline: return
promise = asyncio.run_coroutine_threadsafe(self.stop_and_sync(), self.loop)
promise.result()
if not self.connection.closed:
raise Exception('Connection still active')
self.loop.stop()
async def stop_and_sync(self):
self.stopping = True
# todo, assign correct status depending on python exit code
# done = 150, //when all tasks are done
# aborted = 200, //when at least one task aborted
# failed = 250, //when at least one task failed
# crashed = 300, //when at least one task crashed
self.patches['status'] = 150
self.patches['ended'] = time.time() * 1000
self.patches['tasks.main.ended'] = time.time() * 1000
# done = 500,
# aborted = 550,
# failed = 600,
# crashed = 650,
self.patches['tasks.main.status'] = 500
self.patches['tasks.main.instances.0.ended'] = time.time() * 1000
# done = 500,
# aborted = 550,
# failed = 600,
# crashed = 650,
self.patches['tasks.main.instances.0.status'] = 500
while len(self.patches) > 0 or len(self.queue) > 0:
await asyncio.sleep(0.15)
await self.connection.close()
async def register_controller(self, name: str, controller):
self.controllers[name] = controller
async def subscriber(message, done):
if message['type'] == 'error':
done()
del self.controllers[name]
raise Exception('Register controller error: ' + message['error'])
if message['type'] == 'ack':
pass
if message['type'] == 'peerController/message':
data = message['data']
if not hasattr(controller, data['action']):
error = f"Requested action {message['action']} not available in {name}"
print(error, file=sys.stderr)
await self._message({
'name': 'peerController/message',
'controllerName': name,
'replyId': message['replyId'],
'data': {'type': 'error', 'id': 0, 'stack': None, 'entityName': '@error:default',
'error': error}
}, no_response=True)
if data['name'] == 'actionTypes':
parameters = []
for arg in inspect.getfullargspec(getattr(controller, data['action'])).args:
parameters.append({
'type': 'Any',
'array': False,
'partial': False
})
await self._message({
'name': 'peerController/message',
'controllerName': name,
'replyId': message['replyId'],
'data': {
'type': 'actionTypes/result',
'id': 0,
'parameters': parameters,
'returnType': {'partial': False, 'type': 'Any', 'array': False}
}
}, no_response=True)
if data['name'] == 'action':
try:
res = getattr(controller, data['action'])(*data['args'])
await self._message({
'name': 'peerController/message',
'controllerName': name,
'replyId': message['replyId'],
'data': {
'type': 'next/json',
'id': message['id'],
'next': res,
}
}, no_response=True)
except Exception as e:
await self._message({
'name': 'peerController/message',
'controllerName': name,
'replyId': message['replyId'],
'data': {'type': 'error', 'id': 0, 'stack': None, 'entityName': '@error:default',
'error': str(e)}
}, no_response=True)
await self._subscribe({
'name': 'peerController/register',
'controllerName': name,
}, subscriber)
class Controller:
def __init__(self, client):
self.client = client
def stop(self):
self.client._message({
'name': 'peerController/unregister',
'controllerName': name,
})
return Controller(self)
async def _action(self, controller: str, action: str, args: List, lock=True):
if lock: await self.connecting
if self.offline: return
if self.stopping: raise Exception('In shutdown: actions disallowed')
res = await self._message({
'name': 'action',
'controller': controller,
'action': action,
'args': args,
'timeout': 60
}, lock=lock)
if res['type'] == 'next/json':
return res['next']
if res['type'] == 'error':
print(res, file=sys.stderr)
raise Exception('API Error: ' + res['error'])
raise Exception(f"Invalid action type '{res['type']}'. Not implemented")
def job_action(self, action: str, args: List):
return asyncio.run_coroutine_threadsafe(self._action('job', action, args), self.loop)
async def _subscribe(self, message, subscriber):
await self.connecting
self.message_id += 1
message['id'] = self.message_id
message_id = self.message_id
def on_done():
del self.subscriber[message_id]
async def on_incoming_message(incoming_message):
await subscriber(incoming_message, on_done)
self.subscriber[self.message_id] = on_incoming_message
self.queue.append(message)
async def _message(self, message, lock=True, no_response=False):
if lock: await self.connecting
self.message_id += 1
message['id'] = self.message_id
if not no_response:
self.callbacks[self.message_id] = self.loop.create_future()
self.queue.append(message)
if no_response:
return
return await self.callbacks[self.message_id]
def patch(self, path: str, value: any):
if self.offline: return
if self.stopping: raise Exception('In shutdown: patches disallowed')
self.patches[path] = value
async def send_messages(self, connection):
while not connection.closed:
try:
q = self.queue[:]
for m in q:
await connection.send(json.dumps(m))
self.queue.remove(m)
except Exception:
return
if len(self.patches) > 0:
try:
send = self.patches.copy()
await connection.send(json.dumps({
'name': 'action',
'controller': 'job',
'action': 'patchJob',
'args': [
send
],
'timeout': 60
}))
for i in send.keys():
if self.patches[i] == send[i]:
del self.patches[i]
except websockets.exceptions.ConnectionClosed:
return
await asyncio.sleep(0.25)
async def handle_messages(self, connection):
while not connection.closed:
try:
res = json.loads(await connection.recv())
except websockets.exceptions.ConnectionClosedError:
# we need reconnect
break
except websockets.exceptions.ConnectionClosedOK:
# we closed on purpose, so no reconnect necessary
return
if res and 'id' in res:
if res['id'] in self.subscriber:
await self.subscriber[res['id']](res)
if res['id'] in self.callbacks:
self.callbacks[res['id']].set_result(res)
del self.callbacks[res['id']]
if not self.stopping:
print("Deepkit: lost connection. reconnect ...")
self.connecting = self.loop.create_future()
self.connected.on_next(False)
self.loop.create_task(self._connect())
async def _connect_job(self, host: str, port: int, id: str, token: str):
try:
self.connection = await websockets.connect(f"ws://{host}:{port}")
except Exception:
# try again later
await asyncio.sleep(1)
self.loop.create_task(self._connect())
return
self.loop.create_task(self.handle_messages(self.connection))
self.loop.create_task(self.send_messages(self.connection))
res = await self._message({
'name': 'authenticate',
'token': {
'id': 'job',
'token': token,
'job': id
}
}, lock=False)
if not res['result'] or res['result'] is not True:
raise Exception('Job token invalid')
# load job controller
await self._message({
'name': 'action',
'controller': 'job',
'action': ''
}, lock=False)
self.connecting.set_result(True)
if self.connections > 0:
print("Deepkit: Reconnected.")
self.connected.on_next(True)
self.connections += 1
async def _connect(self):
# we want to restart with a empty queue, so authentication happens always first
queue_copy = self.queue[:]
self.queue = []
if self.token:
await self._connect_job(self.host, self.port, self.job_id, self.token)
else:
account_config = self.get_account_config(self.account)
self.host = account_config['host']
self.port = account_config['port']
try:
self.connection = await websockets.connect(f"ws://{self.host}:{self.port}")
except Exception as e:
self.offline = True
print(f"Deepkit: App not started or server not reachable. Monitoring disabled. {e}")
self.connecting.set_result(False)
return
self.loop.create_task(self.handle_messages(self.connection))
self.loop.create_task(self.send_messages(self.connection))
res = await self._message({
'name': 'authenticate',
'token': {
'id': 'user',
'token': account_config['token']
}
}, lock=False)
if not res['result']:
raise Exception('Login invalid')
link = self.get_folder_link()
deepkit_config_yaml = None
if self.config_path and os.path.exists(self.config_path):
deepkit_config_yaml = open(self.config_path, 'r', encoding='utf-8').read()
job = await self._action('app', 'createJob', [link['projectId'], self.config_path, deepkit_config_yaml],
lock=False)
deepkit.globals.loaded_job = job
self.token = await self._action('app', 'getJobAccessToken', [job['id']], lock=False)
self.job_id = job['id']
await self.connection.close()
await self._connect_job(self.host, self.port, self.job_id, self.token)
self.queue = queue_copy + self.queue
def get_account_config(self, name: str) -> Dict:
with open(os.path.expanduser('~') + '/.deepkit/config', 'r') as h:
config = json.load(h)
for account in config['accounts']:
if account['name'] == name:
return account
raise Exception(f"No account for {name} found.")
def get_folder_link(self) -> Dict:
with open(os.path.expanduser('~') + '/.deepkit/config', 'r') as h:
config = json.load(h)
for link in config['folderLinks']:
if is_in_directory(sys.path[0], link['path']):
return link
raise Exception(f"No project link for {sys.path[0]} found.")
class LobbyConnection:
"""
GUARANTEES:
- After calling connect, connection will leave disconnected state.
"""
def __init__(self, host, port):
self.socket = QtNetwork.QTcpSocket()
self.socket.stateChanged.connect(self._on_socket_state_change)
self.socket.readyRead.connect(self.read)
self.socket.error.connect(self._on_error)
self.socket.setSocketOption(QtNetwork.QTcpSocket.KeepAliveOption, 1)
self._host = host
self._port = port
self.block_size = 0
self._obs_state = BehaviorSubject(ConnectionState.DISCONNECTED)
self.obs_state = self._obs_state.pipe(ops.distinct_until_changed())
self.message_stream = Subject()
def _socket_state(self, s):
qas = QtNetwork.QAbstractSocket
if s in [qas.ConnectedState, qas.ClosingState]:
return ConnectionState.CONNECTED
elif s in [qas.BoundState, qas.HostLookupState, qas.ConnectingState]:
return ConnectionState.CONNECTING
else:
return ConnectionState.DISCONNECTED
@property
def state(self):
return self._obs_state.value
def _on_socket_state_change(self, state):
s = self._socket_state(state)
if s is ConnectionState.DISCONNECTED:
self.block_size = 0
self._obs_state.on_next(s)
# Conflict with PySide2 signals!
# Idempotent
def connect_(self):
if self.state is not ConnectionState.DISCONNECTED:
return
self.socket.connectToHost(self._host, self._port)
def disconnect_(self):
self.socket.disconnectFromHost()
def read(self):
ins = QtCore.QDataStream(self.socket)
ins.setVersion(QtCore.QDataStream.Qt_4_2)
while not ins.atEnd():
if self.block_size == 0:
if self.socket.bytesAvailable() < 4:
return
self.block_size = ins.readUInt32()
if self.socket.bytesAvailable() < self.block_size:
return
data = ins.readQString()
self.block_size = 0
self.message_stream.on_next(data)
def write(self, data):
block = QtCore.QByteArray()
out = QtCore.QDataStream(block, QtCore.QIODevice.ReadWrite)
out.setVersion(QtCore.QDataStream.Qt_4_2)
out.writeUInt32(2 * len(data) + 4)
out.writeQString(data)
if self.socket.state() == QtNetwork.QAbstractSocket.ConnectedState:
self.socket.write(block)
def _on_error(self):
self.disconnect_()