def test_acquire_timeout(self):
lock = locks.Lock()
lock.acquire()
with self.assertRaises(gen.TimeoutError):
yield lock.acquire(timeout=timedelta(seconds=0.01))
# Still locked.
self.assertFalse(lock.acquire().done())
def __init__(self, scope):
super(BokehAppWebsocketConsumer, self).__init__(scope)
kwargs = self.scope['url_route']["kwargs"]
self.application_context = kwargs["application_context"]
self.application_context._loop = asyncio.get_event_loop()
self.bokeh_websocket_path = kwargs["bokeh_websocket_path"]
self._clients = set()
self.lock = locks.Lock()
def __init__(self, kernel_manager, kernel_name=None):
self.kernel_manager = kernel_manager
self.kernel_name = kernel_name
self.current_iopub_handler = None
self.installed_modules = []
self.app_stats = None
self.run_stats = None
self.lock = locks.Lock()
self.kernel_handle = None
def __init__(self, tornado_app, *args, **kw):
self.receiver = None
self.handler = None
self.connection = None
self.application_context = kw['application_context']
self.latest_pong = -1
# write_lock allows us to lock the connection to send multiple
# messages atomically.
self.write_lock = locks.Lock()
# Note: tornado_app is stored as self.application
super(WSHandler, self).__init__(tornado_app, *args, **kw)
def __init__(self):
"""Constructor."""
static_dir = _get_static_dir()
static_files, md5 = _get_static_files(static_dir)
self._static_dir = static_dir
self._static_files = static_files
self._release_hash = '%s-%s' % (streamlit.__version__,
base58.b58encode(
md5.digest()[:3]).decode("utf-8"))
self._write_lock = locks.Lock()
def __init__(self, scope: Dict[str, Any]) -> None:
super(WSConsumer, self).__init__(scope)
kwargs = self.scope['url_route']["kwargs"]
self.application_context = kwargs["app_context"]
if self.application_context.io_loop is None:
raise RuntimeError("io_loop should already been set")
self._clients = set()
self.lock = locks.Lock()
def __init__(self):
# The lock is needed so we don't let two coroutines write
# simultaneously to a pipeline; then the first call to read might
# read translations of text put there by the second call …
self.lock = locks.Lock()
# The users count is how many requests have picked this
# pipeline for translation. If this is 0, we can safely shut
# down the pipeline.
self.users = 0
self.lastUsage = 0
self.useCount = 0
class ConfigHandler(tornado.web.RequestHandler):
load_config_lock = locks.Lock()
def initialize(self, config_dir, models, **kwargs):
self.config_dir = config_dir
self.models = models
@tornado.web.asynchronous
@tornado.gen.coroutine
def post(self, *args, **kwargs):
"""
Handles the creation of a new engine based on a slightly modified config, supplied
via HTTP Post. The request requires 1 URL parameter: new_config_url, which points
to a URL (can be local) from where to download the config. This call is blocking.
WARNING All configs must have the following added:
extract_output_layer_name = "<OUTPUT LAYER ID>"
:param args:
:param kwargs:
:return: ASCII encoded hash of the new engine based on the provided config.
"""
data = json.loads(self.request.body)
config_url = data["new_config_url"]
print('Received request to load config %s' % config_url, file=log.v3)
hash_engine = hashlib.new('ripemd160')
hash_engine.update(config_url.encode('utf8'))
hash_val = hash_engine.hexdigest()
with (yield self.load_config_lock.acquire()):
if hash_val in self.models:
print('Using existing model with hash %s' % hash_val,
file=log.v3)
self.write(hash_val)
return
# Download new config file and save to temp folder.
print('loading config %s' % config_url, file=log.v5)
config_file = os.path.join(self.config_dir, hash_val + ".config")
try:
urllib.request.urlretrieve(config_url, config_file)
except (urllib.request.URLError, urllib.request.HTTPError) as e:
self.write('Error: Loading in config file from URL %s' % e)
return
try:
self.models[hash_val] = Model(config_file)
except Exception as e:
print('Error: loading model failed %s' % str(e), file=log.v1)
self.set_status(499, str(e))
else:
self.write(hash_val)
def __init__(self, session_id, document):
if session_id is None:
raise ValueError("Sessions must have an id")
if document is None:
raise ValueError("Sessions must have a document")
self._id = session_id
self._document = document
self._subscribed_connections = set()
self._last_unsubscribe_time = current_time()
self._lock = locks.Lock()
self._current_patch = None
self._current_patch_connection = None
self._document.on_change(self._document_changed)
class ManagerClient(object):
process_list = []
process_dict = {}
write_lock = locks.Lock()
_instance = None
def __init__(self, worker_num=1):
if ManagerClient._instance is None:
self.worker_num = worker_num if worker_num > 0 else 1
LOG.debug("ManagerClient, worker_num: %s", self.worker_num)
pipe_master, pipe_client = Pipe()
p = Manager(pipe_client, self.worker_num, CONFIG)
p.daemon = True
ManagerClient.process_list.append(p)
ManagerClient.process_dict["manager"] = [p, pipe_master]
p.start()
ManagerClient._instance = self
else:
self.worker_num = ManagerClient._instance.worker_num
@gen.coroutine
def check_app(self, app_id):
result = False
LOG.debug("start check app, app_id: %s", app_id)
with (yield ManagerClient.write_lock.acquire()):
LOG.debug("get check app lock, app_id: %s", app_id)
ManagerClient.process_dict["manager"][1].send(
(Command.check_app, app_id))
LOG.debug("send check app, app_id: %s", app_id)
while not ManagerClient.process_dict["manager"][1].poll():
yield gen.moment
LOG.debug("recv check app, app_id: %s", app_id)
r = ManagerClient.process_dict["manager"][1].recv()
LOG.debug("end check app, app_id: %s, r: %s", app_id, r)
if r[1]:
result = r[1]
raise gen.Return(result)
def close(self):
try:
LOG.info("close ManagerClient")
ManagerClient.process_dict["manager"][1].send((Command.exit, None))
for p in ManagerClient.process_list[1:]:
p.terminate()
for p in ManagerClient.process_list:
while p.is_alive():
time.sleep(0.5)
LOG.debug("sleep 0.5 second")
LOG.info("All Process Exit!")
except Exception as e:
LOG.exception(e)
def start(self):
self.kernel_id = self.kernel_manager.start_kernel().result()
kernel = self.kernel_manager.get_kernel(self.kernel_id)
self.kernel_client = kernel.client()
self.kernel_client.session = type(self.kernel_client.session)(
config=kernel.session.config, key=kernel.session.key)
self.iopub = self.kernel_manager.connect_iopub(self.kernel_id)
# Start channels and wait for ready
self.kernel_client.start_channels()
self.kernel_client.wait_for_ready()
print("kernel client initialized")
self.lock = locks.Lock()
#Initialize PixieDust
future = self.execute_code(
"""
import pixiedust
import pkg_resources
import json
from pixiedust.display.app import pixieapp
class Customizer():
def __init__(self):
self.gateway = 'true'
def customizeOptions(self, options):
options.update( {'cell_id': 'dummy', 'showchrome':'false', 'gateway':self.gateway})
options.update( {'nostore_pixiedust': 'true'})
pixieapp.pixieAppRunCustomizer = Customizer()
print(json.dumps( {"installed_modules": list(pkg_resources.AvailableDistributions())} ))
""", lambda acc: json.dumps([
msg['content']['text'] for msg in acc
if msg['header']['msg_type'] == 'stream'
],
default=self._date_json_serializer))
def done(fut):
results = json.loads(fut.result())
for result in results:
try:
val = json.loads(result)
if isinstance(val, dict) and "installed_modules" in val:
self.installed_modules = val["installed_modules"]
break
except:
pass
app_log.debug("Installed modules %s", self.installed_modules)
future.add_done_callback(done)
def start(args):
global cozmoBlockly, nodejs
app = CozmoBlockly([
(r'/cozmo/()', CozmoBlockly.HomeHandler, dict(args=args)),
(r'/cozmo/(.+)', tornado.web.StaticFileHandler
if not args.dev else CozmoBlockly.NoCacheStaticFileHandler,
dict(path='../cozmo-blockly')),
(r'/blockly/(.*)', tornado.web.StaticFileHandler
if not args.dev else CozmoBlockly.NoCacheStaticFileHandler,
dict(path='../blockly')),
(r'/closure-library/(.*)', tornado.web.StaticFileHandler
if not args.dev else CozmoBlockly.NoCacheStaticFileHandler,
dict(path='../closure-library')),
(r'/(saves)/(.*)', CozmoBlockly.SavesHandler),
(r'/robot/submit', CozmoBlockly.RobotSubmitHandler),
(r'/robot/terminate', CozmoBlockly.RobotTerminateHandler),
(r'/camSub', CozmoBlockly.WSCameraSubHandler),
(r'/camPub', CozmoBlockly.WSCameraPubHandler),
(r'/WsSub', CozmoBlockly.WS3dSubHandler),
(r'/WsPub', CozmoBlockly.WS3dPubHandler),
])
cozmoBlockly = app
if not args.nonsecure:
try:
nodejs = Popen(['node', '../nodejs/headless.js'])
except FileNotFoundError as e:
print("node.js wasn't found in your system.")
print(
"Secure mode (default) requires node.js. Please install node.js and follow README instructions."
)
return
print('[Server] Starting server...')
tornado.platform.asyncio.AsyncIOMainLoop().install()
if args.dev:
print('[Server] Running in debug mode')
app.listen(9090)
app._executor = CodeExecutor(args.nonsecure, args.nocozmo, args.aruco)
app._lock = locks.Lock()
app._wsCamera = None
app._ws3d = None
app._ioloop = tornado.ioloop.IOLoop.current()
app._ioloop.start()
print('[Server] Server stopped')
def test_acquire_fifo_async_with(self):
# Repeat the above test using `async with lock:`
# instead of `with (yield lock.acquire()):`.
lock = locks.Lock()
self.assertTrue(lock.acquire().done())
N = 5
history = []
async def f(idx):
async with lock:
history.append(idx)
futures = [f(i) for i in range(N)]
lock.release()
yield futures
self.assertEqual(list(range(N)), history)
def __init__(self,
connection,
loop,
future_store,
channel_number=None,
publisher_confirms=True,
on_return_raises=False):
"""
Create a new instance of the Channel. Don't call this directly, this should
be constructed by the connection.
:type connection: :class:`pika.adapters.tornado_connection.TornadoConnection`
:type loop: :class:`tornado.ioloop.IOLoop`
:param future_store: The future store to use
:type future_store: :class:`topika.common.FutureStore`
:type channel_number: int
:type publisher_confirms: bool
:type on_return_raises: bool
"""
super(Channel, self).__init__(loop, future_store.create_child())
self._channel = None # type: pika.channel.Channel
self._connection = connection
self._confirmations = {}
self._on_return_callbacks = []
self._delivery_tag = 0
self._write_lock = locks.Lock()
self._channel_number = channel_number
self._publisher_confirms = publisher_confirms
if not publisher_confirms and on_return_raises:
raise RuntimeError(
'on_return_raises must be uses with publisher confirms')
self._on_return_raises = on_return_raises
self.default_exchange = self.EXCHANGE_CLASS(
loop=self.loop,
future_store=self._futures.create_child(),
channel=self._channel,
publish_method=self._publish,
name='',
type=exchange.ExchangeType.DIRECT,
passive=None,
durable=None,
auto_delete=None,
internal=None,
arguments=None)
def test_acquire_fifo(self):
lock = locks.Lock()
self.assertTrue(lock.acquire().done())
N = 5
history = []
@gen.coroutine
def f(idx):
with (yield lock.acquire()):
history.append(idx)
futures = [f(i) for i in range(N)]
self.assertFalse(any(future.done() for future in futures))
lock.release()
yield futures
self.assertEqual(list(range(N)), history)
def __init__(self, tornado_app, *args, **kw) -> None:
self.receiver = None
self.handler = None
self.connection = None
self.application_context = kw['application_context']
self.latest_pong = -1
# write_lock allows us to lock the connection to send multiple
# messages atomically.
self.write_lock = locks.Lock()
self._token = None
self._compression_level = kw.pop('compression_level', None)
self._mem_level = kw.pop('mem_level', None)
# Note: tornado_app is stored as self.application
super().__init__(tornado_app, *args, **kw)
def __init__(self,
host=DEFAULT_HOST,
port=DEFAULT_PORT,
db=DEFAULT_DB,
on_close=None):
self._buffer = bytes()
self._busy = locks.Lock()
self._client = tcpclient.TCPClient()
self._connecting = None
self._default_db = int(db or DEFAULT_DB)
self._host = host
self._port = port
self._ioloop = ioloop.IOLoop.current()
self._on_close = on_close
self._pool = []
self._reader = hiredis.Reader()
self._stream = None
super(RedisClient, self).__init__()
async def device_watch(wda_directory: str):
"""
When iOS device plugin, launch WDA
"""
lock = locks.Lock() # WDA launch one by one
async for event in idb.track_devices():
if event.udid.startswith("ffffffffffffffffff"):
logger.debug("Invalid event: %s", event)
continue
logger.debug("Event: %s", event)
if event.present:
d = idb.WDADevice(event.udid, lock=lock, callback=_device_callback)
d.wda_directory = wda_directory
idevices[event.udid] = d
d.start()
else: # offline
await idevices[event.udid].stop()
idevices.pop(event.udid)
async def device_watch():
"""
When iOS device plugin, launch WDA
"""
lock = locks.Lock() # WDA launch one by one
async for event in idb.track_devices():
if event.udid.startswith("ffffffffffffffffff"):
logger.debug("Invalid event: %s", event)
continue
logger.debug("Event: %s", event)
if event.present:
idevices[event.udid] = d = idb.IDevice(event.udid, lock=lock)
IOLoop.current().spawn_callback(d.run_wda_forever,
partial(_device_callback, d))
else: # offline
await idevices[event.udid].stop()
idevices.pop(event.udid)
def __init__(self, session_id, document, io_loop=None):
if session_id is None:
raise ValueError("Sessions must have an id")
if document is None:
raise ValueError("Sessions must have a document")
self._id = session_id
self._document = document
self._loop = io_loop
self._subscribed_connections = set()
self._last_unsubscribe_time = current_time()
self._lock = locks.Lock()
self._current_patch = None
self._current_patch_connection = None
self._document.on_change_dispatch_to(self)
self._callbacks = _DocumentCallbackGroup(io_loop)
self._pending_writes = None
wrapped_callbacks = self._wrap_session_callbacks(
self._document.session_callbacks)
self._callbacks.add_session_callbacks(wrapped_callbacks)
def _run(self):
"""Run method that can be profiled"""
self.set_state(self.STATE_INITIALIZING)
self.ioloop = ioloop.IOLoop.current()
self.consumer_lock = locks.Lock()
self.sentry_client = self.setup_sentry(self._kwargs['config'],
self.consumer_name)
try:
self.setup()
except (AttributeError, ImportError):
return self.on_startup_error(
'Failed to import the Python module for {}'.format(
self.consumer_name))
if not self.is_stopped:
try:
self.ioloop.start()
except KeyboardInterrupt:
LOGGER.warning('CTRL-C while waiting for clean shutdown')
def __init__(self, server, closesessionsonstart=True, fact_back_off=60):
self._server = server
self._agent_procs = {} # env uuid -> subprocess.Popen
server.add_future(self.start_agents())
# back-off timer for fact requests
self._fact_resource_block = fact_back_off
# per resource time of last fact request
self._fact_resource_block_set = {}
self._server_storage = server._server_storage
# session lock
self.session_lock = locks.Lock()
# all sessions
self.sessions = {}
# live sessions
self.tid_endpoint_to_session = {}
self.closesessionsonstart = closesessionsonstart
def __init__(self, loop, future_store, channel, name, durable, exclusive,
auto_delete, arguments):
"""
:type loop: :class:`tornado.ioloop.IOLoop`
:type future_store: :class:`topika.FutureStore`
:type channel: :class:`pika.Channel`
:type name: str
:type durable: bool
:type exclusive: bool
:type auto_delete: bool
:type arguments: dict
"""
super(Queue, self).__init__(loop, future_store)
self._channel = channel
self.name = name or ''
self.durable = durable
self.exclusive = exclusive
self.auto_delete = auto_delete
self.arguments = arguments
self.declaration_result = None # type: DeclarationResult
self._get_lock = locks.Lock()
def __init__(self, session_id: ID, document: Document, io_loop: IOLoop | None = None, token: str | None = None) -> None:
if session_id is None:
raise ValueError("Sessions must have an id")
if document is None:
raise ValueError("Sessions must have a document")
self._id = session_id
self._token = token
self._document = document
self._loop = io_loop
self._subscribed_connections = set()
self._last_unsubscribe_time = current_time()
self._lock = locks.Lock()
self._current_patch_connection = None
self._document.callbacks.on_change_dispatch_to(self)
self._callbacks = DocumentCallbackGroup(io_loop)
self._pending_writes = None
self._destroyed = False
self._expiration_requested = False
self._expiration_blocked_count = 0
wrapped_callbacks = [self._wrap_session_callback(cb) for cb in self._document.session_callbacks]
self._callbacks.add_session_callbacks(wrapped_callbacks)
def __init__(self, session_id, document, io_loop=None):
if session_id is None:
raise ValueError("Sessions must have an id")
if document is None:
raise ValueError("Sessions must have a document")
self._id = session_id
self._document = document
self._loop = io_loop
self._subscribed_connections = set()
self._last_unsubscribe_time = current_time()
self._lock = locks.Lock()
self._current_patch = None
self._current_patch_connection = None
self._document.on_change(self._document_changed)
self._callbacks = {}
self._pending_writes = None
for cb in self._document.session_callbacks:
if isinstance(cb, PeriodicCallback):
self._add_periodic_callback(cb)
elif isinstance(cb, TimeoutCallback):
self._add_timeout_callback(cb)
def __init__(self):
self._runner_process = None
self._control_process = None
self._watchdog = ProcessWatchdog(config.Watchdog.CHECK_INTERVAL)
self.push_messages_receiver = PushMessageReceiver()
self.config_builder = ConfigurationBuilder(config.Engine.CONFIGURATION_BUILDER)
self.message_handler = MessageHandler(self)
self.message_sender = MessageSender()
self.message_router = MessageRouter(self.message_sender, self.message_handler.default_message_handler)
self.state = ManagerState.EMPTY
self._http_client = httpclient.HTTPClient()
self._alert_registered = False
self.obsi_id = getnode() # A unique identifier of this OBSI
self._engine_running = False
self._engine_running_lock = locks.Lock()
self._processing_graph_set = False
self._engine_config_builder = None
self._keep_alive_periodic_callback = None
self._avg_cpu = 0
self._avg_duration = 0
self._supported_elements_types = []
self._alert_messages_handler = None
self._log_messages_handler = None
def __init__(self, name, test_tools_addr, stop_timeout):
# параметр для управления контейнером
self.name = name
# параметры для коммуникации с test-tools внутри контейнера
self.test_tools_addr = test_tools_addr
self.test_tools_port = None
self.uni_port = None
# динамические параметры этого приложения
self.queue = None
self.docker = Client(base_url='unix:///var/run/docker.sock')
self.stop_future = None
self.stop_timeout = stop_timeout
self.lock = locks.Lock()
# обновляемые параметры
self.db_addr = None
self.running = None
self.tomcat_returncode = None
self.last_task = None
self.last_error = None
self.active_task = None
self.uni_version = None
async def device_watch(wda_directory: str, manually_start_wda: bool,
use_tidevice: bool, wda_bundle_pattern: bool):
"""
When iOS device plugin, launch WDA
"""
lock = locks.Lock() # WDA launch one by one
async for event in idb.track_devices():
if event.udid.startswith("ffffffffffffffffff"):
logger.debug("Invalid event: %s", event)
continue
logger.debug("Event: %s", event)
if event.present:
d = idb.WDADevice(event.udid, lock=lock, callback=_device_callback)
d.wda_directory = wda_directory
d.manually_start_wda = manually_start_wda
d.use_tidevice = use_tidevice
d.wda_bundle_pattern = wda_bundle_pattern
idevices[event.udid] = d
d.start()
else: # offline
await idevices[event.udid].stop()
idevices.pop(event.udid)
def __init__(self,
hosts,
on_close=None,
io_loop=None,
clustering=False,
auto_connect=True):
"""Create a new instance of the ``Client`` class.
:param hosts: A list of host connection values.
:type hosts: list(dict)
:param io_loop: Override the current Tornado IOLoop instance
:type io_loop: tornado.ioloop.IOLoop
:param method on_close: The method to call if the connection is closed
:param bool clustering: Toggle the cluster support in the client
:param bool auto_connect: Toggle the auto-connect on creation feature
"""
self._buffer = bytes()
self._busy = locks.Lock()
self._closing = False
self._cluster = {}
self._clustering = clustering
self._connected = locks.Event()
self._connect_future = concurrent.Future()
self._connection = None
self._discovery = False
self._hosts = hosts
self._on_close_callback = on_close
self._reader = hiredis.Reader()
self.io_loop = io_loop or ioloop.IOLoop.current()
if not self._clustering:
if len(hosts) > 1:
raise ValueError('Too many hosts for non-clustering mode')
if auto_connect:
LOGGER.debug('Auto-connecting')
self.connect()
def __init__(self, config_file):
self.lock = locks.Lock()
self.classification_queue = Queue()
print('loading config %s' % config_file, file=log.v5)
# Load and setup config
try:
self.config = Config.Config()
self.config.load_file(config_file)
self.pause_after_first_seq = self.config.float(
'pause_after_first_seq', 0.2)
self.batch_size = self.config.int('batch_size', 5000)
self.max_seqs = self.config.int('max_seqs', -1)
except Exception:
print('Error: loading config %s failed' % config_file, file=log.v1)
raise
try:
self.devices = self._init_devices()
except Exception:
print('Error: Loading devices for config %s failed' % config_file,
file=log.v1)
raise
print('Starting engine for config %s' % config_file, file=log.v5)
self.engine = Engine.Engine(self.devices)
try:
self.engine.init_network_from_config(config=self.config)
except Exception:
print('Error: Loading network for config %s failed' % config_file,
file=log.v1)
raise
IOLoop.current().spawn_callback(self.classify_in_background)
self.last_used = datetime.datetime.now()