class GConnection(Async):
def __init__(self, *args, **kwargs):
"""
This class is a 'GEvent'-optimized subclass of libcouchbase
which utilizes the underlying IOPS structures and the gevent
event primitives to efficiently utilize couroutine switching.
"""
experimental.enabled_or_raise()
super(GConnection, self).__init__(IOPS(), *args, **kwargs)
def _do_ctor_connect(self):
if self.connected:
return
self._connect()
self._evconn = AsyncResult()
self._conncb = self._on_connected
self._evconn.get()
self._evconn = None
def _on_connected(self, err):
if err:
self._evconn.set_exception(err)
else:
self._evconn.set(None)
def _waitwrap(self, cbasync):
cur_thread = getcurrent()
cbasync.callback = cur_thread.switch
cbasync.errback = lambda r, x, y, z: cur_thread.throw(x, y, z)
return get_hub().switch()
def _meth_factory(meth, name):
def ret(self, *args, **kwargs):
return self._waitwrap(meth(self, *args, **kwargs))
return ret
def _http_request(self, **kwargs):
res = super(GConnection, self)._http_request(**kwargs)
if kwargs.get('chunked', False):
return res #views
e = Event()
res._callback = lambda x, y: e.set()
e.wait()
res._maybe_raise()
return res
def query(self, *args, **kwargs):
kwargs['itercls'] = GView
ret = super(GConnection, self).query(*args, **kwargs)
ret.start()
return ret
locals().update(Async._gen_memd_wrappers(_meth_factory))
class DataMonitor(object):
_STOP_REQUEST = object()
def __init__(self, client, path, callback, args, kwargs):
self.client = client
self.path = path
self.callback = callback
self.args = args
self.kwargs = kwargs
self.started = AsyncResult()
self.queue = Queue()
self._delay = 1.343
self.max_delay = 180
def _monitor(self):
"""Run the monitoring loop."""
def watcher(event):
self.queue.put(event)
while True:
try:
data, stat = self.client.get(self.path, watcher)
except zookeeper.NoNodeException:
if not self.started.ready():
self.started.set(None)
gevent.sleep(1)
continue
except (zookeeper.ConnectionLossException,
zookeeper.SessionExpiredException,
zookeeper.InvalidStateException), err:
if not self.started.ready():
self.started.set_exception(err)
break
logging.error("got %r while monitoring %s", str(err),
self.path)
gevent.sleep(self._delay)
self._delay += self._delay * random.random()
self._delay = min(self._delay, self.max_delay)
continue
except Exception, err:
if not self.started.ready():
self.started.set_exception(err)
break
raise
self.callback(data, *self.args, **self.kwargs)
if not self.started.ready():
self.started.set(None)
event = self.queue.get()
if event is self._STOP_REQUEST:
break
class Task(object):
def __init__(self, _id, func, args, kwargs):
self.id = _id
self.result = AsyncResult()
self.func = func
self.args = args
self.kwargs = kwargs
def execute(self):
try:
self.result.set(self.func(*self.args, **self.kwargs))
except Exception, e:
log.exception('Task.execute: execution failed.')
self.result.set_exception(e)
def test_set_exc(self):
log = []
e = AsyncResult()
def waiter():
try:
result = e.get()
log.append(('received', result))
except Exception as ex:
log.append(('catched', ex))
gevent.spawn(waiter)
obj = Exception()
e.set_exception(obj)
gevent.sleep(0)
assert log == [('catched', obj)], log
def test_set_exc(self):
log = []
e = AsyncResult()
self.assertEqual(e.exc_info, ())
self.assertEqual(e.exception, None)
def waiter():
with self.assertRaises(MyException) as exc:
e.get()
log.append(('caught', exc.exception))
gevent.spawn(waiter)
obj = MyException()
e.set_exception(obj)
gevent.sleep(0)
self.assertEqual(log, [('caught', obj)])
class _SyncCall(object):
def __init__(self, name, *args, **kwargs):
self.name = name
self._args = args
self._kwargs = kwargs
self._result = AsyncResult()
def wait(self, timeout):
return self._result.get(timeout=timeout)
def execute(self, target):
try:
function = getattr(target, self.name)
self._result.set(function(*self._args, **self._kwargs))
except Exception as error:
self._result.set_exception(error)
class FutureResult(object):
"""
Future results for asynchronous operations.
"""
def __init__(self):
self._result = AsyncResult()
self.created_at = time.time()
def get(self, timeout=None):
return self._result.get(block=True, timeout=timeout)
def set(self, value):
self._result.set(value)
def set_exception(self, exception):
self._result.set_exception(exception)
def execution_result(executor):
result = AsyncResult()
deferred = executor()
assert isinstance(deferred, Deferred), 'Another middleware has converted the execution result ' \
'away from a Deferred.'
deferred.add_callbacks(result.set, lambda e: result.set_exception(e.value, (e.type, e.value, e.traceback)))
return result.get()
def test_set_exc(self):
log = []
e = AsyncResult()
def waiter():
try:
result = e.get()
log.append(("received", result))
except Exception as ex:
log.append(("catched", ex))
if PY3:
ex.__traceback__ = None
gevent.spawn(waiter)
obj = Exception()
e.set_exception(obj)
gevent.sleep(0)
assert log == [("catched", obj)], log
class FileRequest(object):
""" File Request Class
Uses gevent's AsyncResult to notify of download completion"""
__slots__ = ('node_address', 'filename', 'result')
def __init__(self, node_address, filename):
self.node_address = node_address
self.filename = filename
self.result = AsyncResult()
def wait(self):
""" block until result set """
self.result.get()
def complete(self):
""" request completed, set result,
causes wait() to return"""
self.result.set()
def exception(self, exception):
""" exception ocurred during request """
self.result.set_exception(exception)
class mock_server(object):
def __init__(self, handler):
self.handler = handler
self.result = AsyncResult()
self.server = StreamServer(('127.0.0.1', 0), self)
def __call__(self, socket, address):
try:
self.result.set(self.handler(socket, address))
except Exception as error:
self.result.set_exception(error)
finally:
socket.close()
def __enter__(self):
self.server.start()
return self.server
def __exit__(self, exc_type, exc_value, traceback):
if exc_type is None:
self.result.get()
self.server.stop()
class DataMonitor(object):
_STOP_REQUEST = object()
def __init__(self, client, path, callback, args, kwargs):
self.client = client
self.path = path
self.callback = callback
self.args = args
self.kwargs = kwargs
self.started = AsyncResult()
self.queue = Queue()
def _monitor(self):
"""Run the monitoring loop."""
def watcher(event):
self.queue.put(event)
while True:
try:
data, stat = self.client.get(self.path, watcher)
except zookeeper.NoNodeException:
if not self.started.ready():
self.started.set(None)
gevent.sleep(1)
continue
except Exception, err:
if not self.started.ready():
self.started.set_exception(err)
break
self.callback(data, *self.args, **self.kwargs)
if not self.started.ready():
self.started.set(None)
event = self.queue.get()
if event is self._STOP_REQUEST:
break
def register_secret_batch(self, secrets: List[typing.Secret]):
secret_batch = list()
secret_registry_transaction = AsyncResult()
for secret in secrets:
secrethash = sha3(secret)
if not self.check_registered(secrethash):
if secret not in self.open_secret_transactions:
secret_batch.append(secret)
self.open_secret_transactions[secret] = secret_registry_transaction
else:
log.info(
'secret already registered',
node=pex(self.node_address),
contract=pex(self.address),
secrethash=encode_hex(secrethash),
)
if not secret_batch:
return
log.info(
'registerSecretBatch called',
node=pex(self.node_address),
contract=pex(self.address),
)
try:
transaction_hash = self._register_secret_batch(secret_batch)
except Exception as e:
secret_registry_transaction.set_exception(e)
raise
else:
secret_registry_transaction.set(transaction_hash)
finally:
for secret in secret_batch:
self.open_secret_transactions.pop(secret, None)
class _Socket(_original_Socket):
"""Green version of :class:`zmq.core.socket.Socket`
The following methods are overridden:
* send
* recv
To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or recieving
is deferred to the hub if a ``zmq.EAGAIN`` (retry) error is raised.
The `__state_changed` method is triggered when the zmq.FD for the socket is
marked as readable and triggers the necessary read and write events (which
are waited for in the recv and send methods).
Some double underscore prefixes are used to minimize pollution of
:class:`zmq.core.socket.Socket`'s namespace.
"""
def __init__(self, context, socket_type):
self.__setup_events()
def close(self, linger=None):
# close the _state_event event, keeps the number of active file descriptors down
if not self._closed and getattr(self, '_state_event', None):
try:
self._state_event.stop()
except AttributeError as e:
# gevent<1.0 compat
self._state_event.cancel()
super(_Socket, self).close(linger)
def __setup_events(self):
self.__readable = AsyncResult()
self.__writable = AsyncResult()
try:
self._state_event = get_hub().loop.io(self.getsockopt(FD), 1) # read state watcher
self._state_event.start(self.__state_changed)
except AttributeError:
# for gevent<1.0 compatibility
from gevent.core import read_event
self._state_event = read_event(self.getsockopt(FD), self.__state_changed, persist=True)
def __state_changed(self, event=None, _evtype=None):
try:
if self.closed:
# if the socket has entered a close state resume any waiting greenlets
self.__writable.set()
self.__readable.set()
return
events = self.getsockopt(zmq.EVENTS)
except ZMQError as exc:
self.__writable.set_exception(exc)
self.__readable.set_exception(exc)
else:
if events & zmq.POLLOUT:
self.__writable.set()
if events & zmq.POLLIN:
self.__readable.set()
def _wait_write(self):
self.__writable = AsyncResult()
self.__writable.get()
def _wait_read(self):
self.__readable = AsyncResult()
self.__readable.get()
def send(self, data, flags=0, copy=True, track=False):
# if we're given the NOBLOCK flag act as normal and let the EAGAIN get raised
if flags & zmq.NOBLOCK:
return super(_Socket, self).send(data, flags, copy, track)
# ensure the zmq.NOBLOCK flag is part of flags
flags |= zmq.NOBLOCK
while True: # Attempt to complete this operation indefinitely, blocking the current greenlet
try:
# attempt the actual call
return super(_Socket, self).send(data, flags, copy, track)
except zmq.ZMQError as e:
# if the raised ZMQError is not EAGAIN, reraise
if e.errno != zmq.EAGAIN:
raise
# defer to the event loop until we're notified the socket is writable
self._wait_write()
def recv(self, flags=0, copy=True, track=False):
if flags & zmq.NOBLOCK:
return super(_Socket, self).recv(flags, copy, track)
flags |= zmq.NOBLOCK
while True:
try:
return super(_Socket, self).recv(flags, copy, track)
except zmq.ZMQError as e:
if e.errno != zmq.EAGAIN:
raise
self._wait_read()
class TaskExecutor(object):
def __init__(self, balancer, index):
self.balancer = balancer
self.index = index
self.task = None
self.proc = None
self.pid = None
self.conn = None
self.state = None
self.key = str(uuid.uuid4())
self.checked_in = Event()
self.result = AsyncResult()
self.exiting = False
self.thread = gevent.spawn(self.executor)
def checkin(self, conn):
self.balancer.logger.debug('Check-in of worker #{0} (key {1})'.format(self.index, self.key))
self.conn = conn
self.state = WorkerState.IDLE
self.checked_in.set()
def get_status(self):
if not self.conn:
return None
try:
st = TaskStatus(0)
if issubclass(self.task.clazz, MasterProgressTask):
progress_subtask_info = self.conn.call_client_sync(
'taskproxy.get_master_progress_info'
)
if progress_subtask_info['increment_progress'] != 0:
progress_subtask_info['progress'] += progress_subtask_info['increment_progress']
progress_subtask_info['increment_progress'] = 0
self.conn.call_client_sync(
'taskproxy.set_master_progress_detail',
{
'progress': progress_subtask_info['progress'],
'increment_progress': progress_subtask_info['increment_progress']
}
)
if progress_subtask_info['active_tids']:
progress_to_increment = 0
concurent_weight = progress_subtask_info['concurent_subtask_detail']['average_weight']
for tid in progress_subtask_info['concurent_subtask_detail']['tids']:
subtask_status = self.balancer.get_task(tid).executor.get_status()
progress_to_increment += subtask_status.percentage * concurent_weight * \
progress_subtask_info['subtask_weights'][str(tid)]
for tid in set(progress_subtask_info['active_tids']).symmetric_difference(
set(progress_subtask_info['concurent_subtask_detail']['tids'])
):
subtask_status = self.balancer.get_task(tid).executor.get_status()
progress_to_increment += subtask_status.percentage * \
progress_subtask_info['subtask_weights'][str(tid)]
progress_subtask_info['progress'] += int(progress_to_increment)
if progress_subtask_info['pass_subtask_details']:
progress_subtask_info['message'] = subtask_status.message
st = TaskStatus(
progress_subtask_info['progress'], progress_subtask_info['message']
)
else:
st.__setstate__(self.conn.call_client_sync('taskproxy.get_status'))
return st
except RpcException as err:
self.balancer.logger.error(
"Cannot obtain status from task #{0}: {1}".format(self.task.id, str(err))
)
self.proc.terminate()
def put_status(self, status):
# Try to collect rusage at this point, when process is still alive
try:
kinfo = bsd.kinfo_getproc(self.pid)
self.task.rusage = kinfo.rusage
except LookupError:
pass
if status['status'] == 'ROLLBACK':
self.task.set_state(TaskState.ROLLBACK)
if status['status'] == 'FINISHED':
self.result.set(status['result'])
if status['status'] == 'FAILED':
error = status['error']
cls = TaskException
if error['type'] == 'task.TaskAbortException':
cls = TaskAbortException
if error['type'] == 'ValidationException':
cls = ValidationException
self.result.set_exception(cls(
code=error['code'],
message=error['message'],
stacktrace=error['stacktrace'],
extra=error.get('extra')
))
def put_warning(self, warning):
self.task.add_warning(warning)
def run(self, task):
self.result = AsyncResult()
self.task = task
self.task.set_state(TaskState.EXECUTING)
filename = None
module_name = inspect.getmodule(task.clazz).__name__
for dir in self.balancer.dispatcher.plugin_dirs:
found = False
try:
for root, _, files in os.walk(dir):
file = first_or_default(lambda f: module_name in f, files)
if file:
filename = os.path.join(root, file)
found = True
break
if found:
break
except FileNotFoundError:
continue
self.conn.call_client_sync('taskproxy.run', {
'id': task.id,
'class': task.clazz.__name__,
'filename': filename,
'args': task.args,
'debugger': task.debugger,
'environment': task.environment
})
try:
self.result.get()
except BaseException as e:
if not isinstance(e, TaskException):
self.balancer.dispatcher.report_error(
'Task {0} raised exception other than TaskException'.format(self.task.name),
e
)
if isinstance(e, TaskAbortException):
self.task.set_state(TaskState.ABORTED, TaskStatus(0, 'aborted'))
else:
self.task.error = serialize_error(e)
self.task.set_state(TaskState.FAILED, TaskStatus(0, str(e), extra={
"stacktrace": traceback.format_exc()
}))
self.task.ended.set()
self.balancer.task_exited(self.task)
self.state = WorkerState.IDLE
return
self.task.result = self.result.value
self.task.set_state(TaskState.FINISHED, TaskStatus(100, ''))
self.task.ended.set()
self.balancer.task_exited(self.task)
self.state = WorkerState.IDLE
def abort(self):
self.balancer.logger.info("Trying to abort task #{0}".format(self.task.id))
# Try to abort via RPC. If this fails, kill process
try:
self.conn.call_client_sync('taskproxy.abort')
except RpcException as err:
self.balancer.logger.warning("Failed to abort task #{0} gracefully: {1}".format(self.task.id, str(err)))
self.balancer.logger.warning("Killing process {0}".format(self.pid))
self.proc.terminate()
def executor(self):
while not self.exiting:
try:
self.proc = Popen(
[TASKWORKER_PATH, self.key],
close_fds=True,
preexec_fn=os.setpgrp,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
self.pid = self.proc.pid
self.balancer.logger.debug('Started executor #{0} as PID {1}'.format(self.index, self.pid))
except OSError:
self.result.set_exception(TaskException(errno.EFAULT, 'Cannot spawn task executor'))
self.balancer.logger.error('Cannot spawn task executor #{0}'.format(self.index))
return
for line in self.proc.stdout:
line = line.decode('utf8')
self.balancer.logger.debug('Executor #{0}: {1}'.format(self.index, line.strip()))
if self.task:
self.task.output += line
self.proc.wait()
if self.proc.returncode == -signal.SIGTERM:
self.balancer.logger.info(
'Executor process with PID {0} was terminated gracefully'.format(
self.proc.pid
)
)
else:
self.balancer.logger.error('Executor process with PID {0} died abruptly with exit code {1}'.format(
self.proc.pid,
self.proc.returncode)
)
self.result.set_exception(TaskException(errno.EFAULT, 'Task executor died'))
gevent.sleep(1)
def die(self):
self.exiting = True
if self.proc:
try:
self.proc.terminate()
except ProcessLookupError:
self.balancer.logger.warning('Executor process with PID {0} already dead'.format(self.proc.pid))
class GeventLoop(object):
def __init__(self):
super(GeventLoop, self).__init__()
self._greenlets = set()
self._idle_handle = 0
self._idle_callbacks = {}
self._exit = AsyncResult()
def _greenlet_spawned(self, greenlet):
greenlet.link(self._greenlet_completed)
greenlet.link_exception(self._greenlet_failed)
self._greenlets.add(greenlet)
return greenlet
def _greenlet_completed(self, greenlet):
self._greenlets.discard(greenlet)
self._entering_idle()
def _greenlet_failed(self, greenlet):
self._exit.set_exception(greenlet.exception)
# alarm
def alarm(self, seconds, callback):
greenlet = gevent.spawn_later(seconds, callback)
return self._greenlet_spawned(greenlet)
def remove_alarm(self, handle):
if handle._start_event.active:
handle._start_event.stop()
return True
return False
# file
def _watch_file(self, fd, callback):
while True:
select.select([fd], [], [])
self._greenlet_spawned(gevent.spawn(callback))
def watch_file(self, fd, callback):
greenlet = gevent.spawn(self._watch_file, fd, callback)
return self._greenlet_spawned(greenlet)
def remove_watch_file(self, handle):
handle.kill()
return True
# idle
def _entering_idle(self):
for callback in self._idle_callbacks.values():
callback()
def enter_idle(self, callback):
self._idle_handle += 1
self._idle_callbacks[self._idle_handle] = callback
return self._idle_handle
def remove_enter_idle(self, handle):
try:
del self._idle_callbacks[handle]
except KeyError:
return False
return True
def run(self):
while True:
greenlets = [self._exit]
greenlets.extend(self._greenlets)
try:
gevent.joinall(greenlets, timeout=1, raise_error=True)
except gevent.Timeout:
pass
self._entering_idle()
class GeventLoop(object):
def __init__(self):
super(GeventLoop, self).__init__()
self._greenlets = set()
self._idle_handle = 0
self._idle_callbacks = {}
self._exit = AsyncResult()
def _greenlet_spawned(self, greenlet):
greenlet.link(self._greenlet_completed)
greenlet.link_exception(self._greenlet_failed)
self._greenlets.add(greenlet)
return greenlet
def _greenlet_completed(self, greenlet):
self._greenlets.discard(greenlet)
self._entering_idle()
def _greenlet_failed(self, greenlet):
self._exit.set_exception(greenlet.exception)
# alarm
def alarm(self, seconds, callback):
greenlet = gevent.spawn_later(seconds, callback)
return self._greenlet_spawned(greenlet)
def remove_alarm(self, handle):
if handle._start_event.active:
handle._start_event.stop()
return True
return False
# file
def _watch_file(self, fd, callback):
while True:
select.select([fd], [], [])
self._greenlet_spawned(gevent.spawn(callback))
def watch_file(self, fd, callback):
greenlet = gevent.spawn(self._watch_file, fd, callback)
return self._greenlet_spawned(greenlet)
def remove_watch_file(self, handle):
handle.kill()
return True
# idle
def _entering_idle(self):
for callback in self._idle_callbacks.values():
callback()
def enter_idle(self, callback):
self._idle_handle += 1
self._idle_callbacks[self._idle_handle] = callback
return self._idle_handle
def remove_enter_idle(self, handle):
try:
del self._idle_callbacks[handle]
except KeyError:
return False
return True
def run(self):
while True:
greenlets = [self._exit]
greenlets.extend(self._greenlets)
try:
gevent.joinall(greenlets, timeout=1, raise_error=True)
except gevent.Timeout:
pass
self._entering_idle()
def register_secret_batch(self, secrets: List[Secret]):
"""Register a batch of secrets. Check if they are already registered at
the given block identifier."""
secrets_to_register = list()
secrethashes_to_register = list()
secrethashes_not_sent = list()
transaction_result = AsyncResult()
wait_for = set()
# secret registration has no preconditions:
#
# - The action does not depend on any state, it's always valid to call
# it.
# - This action is always susceptible to race conditions.
#
# Therefore this proxy only needs to detect if the secret is already
# registered, to avoid sending obviously unecessary transactions, and
# it has to handle race conditions.
with self._open_secret_transactions_lock:
verification_block_hash = self.client.get_confirmed_blockhash()
for secret in secrets:
secrethash = sha3(secret)
secrethash_hex = encode_hex(secrethash)
# Do the local test on `open_secret_transactions` first, then
# if necessary do an RPC call.
#
# The call to `is_secret_registered` has two conflicting
# requirements:
#
# - Avoid sending duplicated transactions for the same lock
# - Operating on a consistent/confirmed view of the blockchain
# (if a secret has been registered in a block that is not
# confirmed it doesn't count yet, an optimization would be to
# *not* send the transaction and wait for the confirmation)
#
# The code below respects the consistent blockchain view,
# meaning that if this proxy method is called with an old
# blockhash an unecessary transaction will be sent, and the
# error will be treated as a race-condition.
other_result = self.open_secret_transactions.get(secret)
if other_result is not None:
wait_for.add(other_result)
secrethashes_not_sent.append(secrethash_hex)
elif not self.is_secret_registered(secrethash,
verification_block_hash):
secrets_to_register.append(secret)
secrethashes_to_register.append(secrethash_hex)
self.open_secret_transactions[secret] = transaction_result
# From here on the lock is not required. Context-switches will happen
# for the gas estimation and the transaction, however the
# synchronization data is limited to the open_secret_transactions
log_details = {
"node": pex(self.node_address),
"contract": pex(self.address),
"secrethashes": secrethashes_to_register,
"secrethashes_not_sent": secrethashes_not_sent,
}
if not secrets_to_register:
log.debug("registerSecretBatch skipped, waiting for transactions",
**log_details)
gevent.joinall(wait_for, raise_error=True)
log.info("registerSecretBatch successful", **log_details)
return
checking_block = self.client.get_checking_block()
gas_limit = self.proxy.estimate_gas(checking_block,
"registerSecretBatch",
secrets_to_register)
receipt = None
transaction_hash = None
msg = None
if gas_limit:
gas_limit = safe_gas_limit(
gas_limit,
len(secrets_to_register) * GAS_REQUIRED_PER_SECRET_IN_BATCH)
log.debug("registerSecretBatch called", **log_details)
try:
transaction_hash = self.proxy.transact("registerSecretBatch",
gas_limit,
secrets_to_register)
self.client.poll(transaction_hash)
receipt = self.client.get_transaction_receipt(transaction_hash)
except Exception as e: # pylint: disable=broad-except
msg = f"Unexpected exception {e} at sending registerSecretBatch transaction."
# Clear `open_secret_transactions` regardless of the transaction being
# successfully executed or not.
with self._open_secret_transactions_lock:
for secret in secrets_to_register:
self.open_secret_transactions.pop(secret)
# As of version `0.4.0` of the contract has *no* asserts or requires.
# Therefore the only reason for the transaction to fail is if there is
# a bug.
unrecoverable_error = (gas_limit is None or receipt is None
or receipt["status"] == RECEIPT_FAILURE_CODE)
exception: Union[RaidenRecoverableError, RaidenUnrecoverableError]
if unrecoverable_error:
# If the transaction was sent it must not fail. If this happened
# some of our assumptions is broken therefore the error is
# unrecoverable
if receipt is not None:
if receipt["gasUsed"] == gas_limit:
# The transaction failed and all gas was used. This can
# happen because of:
#
# - A compiler bug if an invalid opcode was executed.
# - A configuration bug if an assert was executed,
# because version 0.4.0 of the secret registry does not have an
# assert.
# - An ethereum client bug if the gas_limit was
# underestimated.
#
# Safety cannot be guaranteed under any of these cases,
# this error is unrecoverable.
error = (
"Secret registration failed because of a bug in either "
"the solidity compiler, the running ethereum client, or "
"a configuration error in Raiden.")
else:
# The transaction failed and *not* all gas was used. This
# can happen because of:
#
# - A compiler bug if a revert was introduced.
# - A configuration bug, because for 0.4.0 the secret
# registry does not have a revert.
error = (
"Secret registration failed because of a configuration "
"bug or compiler bug. Please double check the secret "
"smart contract is at version 0.4.0, if it is then a "
"compiler bug was hit.")
log.critical(error, **log_details)
exception = RaidenUnrecoverableError(error)
transaction_result.set_exception(exception)
raise exception
# If gas_limit is set and there is no receipt then an exception was
# raised while sending the transaction. This should only happen if
# the account is being used concurrently, which is not supported.
# This can happen because:
#
# - The nonce of the transaction was already used
# - The nonce was reused *and* the account didn't have enough ether
# to pay for the gas
#
# Safety cannot be guaranteed under any of these cases, this error
# is unrecoverable. *Note*: This assumes the ethereum client
# takes into account the current transactions in the pool.
if gas_limit:
assert msg, "Unexpected control flow, an exception should have been raised."
error = (
f"Sending the the transaction for registerSecretBatch failed with: `{msg}`. "
f"This happens if the same ethereum account is being used by more than one "
f"program which is not supported.")
log.critical(error, **log_details)
exception = RaidenUnrecoverableError(error)
transaction_result.set_exception(exception)
raise exception
# gas_limit can fail because:
#
# - The Ethereum client detected the transaction could not
# successfully execute, this happens if an assert/revert is hit.
# - The account is lacking funds to pay for the gas.
#
# Either of these is a bug. The contract does not use
# assert/revert, and the account should always be funded
self.proxy.jsonrpc_client.check_for_insufficient_eth(
transaction_name="registerSecretBatch",
address=self.node_address,
transaction_executed=True,
required_gas=gas_limit,
block_identifier=checking_block,
)
error = "Call to registerSecretBatch couldn't be done"
log.critical(error, **log_details)
exception = RaidenRecoverableError(error)
transaction_result.set_exception(exception)
raise exception
# The local **MUST** transaction_result be set before waiting for the
# other results, otherwise we have a dead-lock
transaction_result.set(transaction_hash)
if wait_for:
log.info("registerSecretBatch waiting for pending", **log_details)
gevent.joinall(wait_for, raise_error=True)
log.info("registerSecretBatch successful", **log_details)
def register_secret_batch(self, secrets: List[Secret]):
secrets_to_register = list()
secrethashes_to_register = list()
secrethashes_not_sent = list()
secret_registry_transaction = AsyncResult()
for secret in secrets:
secrethash = sha3(secret)
secrethash_hex = encode_hex(secrethash)
is_register_needed = (
not self.check_registered(secrethash) and
secret not in self.open_secret_transactions
)
if is_register_needed:
secrets_to_register.append(secret)
secrethashes_to_register.append(secrethash_hex)
self.open_secret_transactions[secret] = secret_registry_transaction
else:
secrethashes_not_sent.append(secrethash_hex)
log_details = {
'node': pex(self.node_address),
'contract': pex(self.address),
'secrethashes': secrethashes_to_register,
'secrethashes_not_sent': secrethashes_not_sent,
}
if not secrets_to_register:
log.debug('registerSecretBatch skipped', **log_details)
return
error_prefix = 'Call to registerSecretBatch will fail'
gas_limit = self.proxy.estimate_gas('pending', 'registerSecretBatch', secrets)
if gas_limit:
error_prefix = 'Call to registerSecretBatch failed'
try:
gas_limit = safe_gas_limit(
gas_limit,
len(secrets) * GAS_REQUIRED_PER_SECRET_IN_BATCH,
)
transaction_hash = self.proxy.transact('registerSecretBatch', gas_limit, secrets)
self.client.poll(transaction_hash)
receipt_or_none = check_transaction_threw(self.client, transaction_hash)
except Exception as e:
secret_registry_transaction.set_exception(e)
msg = 'Unexpected exception at sending registerSecretBatch transaction'
else:
secret_registry_transaction.set(transaction_hash)
finally:
for secret in secrets_to_register:
self.open_secret_transactions.pop(secret, None)
transaction_executed = gas_limit is not None
if not transaction_executed or receipt_or_none:
if transaction_executed:
block = receipt_or_none['blockNumber']
else:
block = 'pending'
self.proxy.jsonrpc_client.check_for_insufficient_eth(
transaction_name='registerSecretBatch',
transaction_executed=transaction_executed,
required_gas=len(secrets) * GAS_REQUIRED_PER_SECRET_IN_BATCH,
block_identifier=block,
)
error_msg = f'{error_prefix}. {msg}'
log.critical(error_msg, **log_details)
raise RaidenUnrecoverableError(error_msg)
log.info('registerSecretBatch successful', **log_details)
class StreamingExtract(object):
def __init__(self, id, hddsem, threadpool):
self.id = id
self.hddsem = hddsem
self.threadpool = threadpool
self.password = None
self.killed = False
self.parts = dict()
self.first = None
self.current = None
self.next = None
self.next_part_event = AsyncResult()
self.rar = None
self.library = None
self._library_added = set()
self._deleted_library = None
extractors[id] = self
def feed_part(self, path, file):
path.finished = AsyncResult()
self.parts[path.path] = path, file
log.debug('fed new part {}: {}'.format(path, path))
if file.state != 'rarextract':
with transaction:
file.state = 'rarextract'
if self.first is None:
self.first = self.current = path, file
self.add_library_files()
self.run(path, file)
else:
if path.path == self.next:
self.next_part_event.set(path)
path.finished.get()
def run(self, path, file):
try:
self.first = self.current
with transaction:
file.greenlet = gevent.getcurrent()
file.on_greenlet_started()
try:
result = self.bruteforce(path, file)
except rarfile.NeedFirstVolume:
self.next = os.path.join(path.dir, "{}.part{}.rar".format(path.basename, "1".zfill(len(path.part))))
self.find_next()
if core.config.delete_extracted_archives:
return False
return
if result and result is not True:
raise result
if self.password:
rarpw = "-p"+self.password
else:
rarpw = "-p-"
cmd = [rarfile.UNRAR_TOOL, "x", "-y", rarpw, "-idq", "-vp", path, file.get_extract_path() + os.sep]
file.log.info("starting extraction of {} with params {}".format(path[1:], cmd))
self.rar = rarfile.custom_popen(cmd)
self.wait_data()
if not path.finished.ready():
path.finished.set()
if core.config.delete_extracted_archives:
return False
except BaseException as e:
traceback.print_exc()
self.kill(e)
raise
def bruteforce(self, path, file):
rar = rarfile.RarFile(path, ignore_next_part_missing=True)
if rar.not_first_volume:
raise rarfile.NeedFirstVolume("First Volume for extraction")
if not rar.needs_password():
self.password = None
return
passwords = []
for i in itertools.chain(file.package.extract_passwords, core.config.bruteforce_passwords):
if not i in passwords:
passwords.append(i)
if rar.needs_password() and rar.infolist():
pw = bruteforce_by_content(rar, passwords)
if not pw:
print "could not find password, asking user"
for pw in file.solve_password(
message="Rarfile {} password cannot be cracked. Enter correct password: #".format(path.name),
retries=5):
pw = bruteforce_by_content(rar, [pw])
if pw:
break
else:
return self.kill('extract password not entered')
else:
print "Found password by content:", pw
self.password = pw
return
print "testing", passwords
if not self.threadpool.apply(bruteforce, (rar, passwords, self.hddsem, file.log)):
# ask user for password
for pw in file.solve_password(
message="Enter the extract password for file: {} #".format(path.name),
retries=5):
if self.threadpool.apply(bruteforce, (rar, [pw], self.hddsem, file.log)):
break
else:
return self.kill('extract password not entered')
self.password = rar._password
if self.password and self.password not in core.config.bruteforce_passwords:
with transaction:
core.config.bruteforce_passwords.append(self.password)
def wait_data(self):
bytes = ''
while True:
data = self.rar.stdout.read(1)
if not data:
break
bytes += data
for i in bytes.splitlines():
if i:
result = self.new_data(i)
if result is True:
bytes = ''
if result and result is not True:
raise result
self.close()
def finish_file(self, path, file):
if file is not None:
with core.transaction:
#if not 'rarextract' in file.completed_plugins:
# file.completed_plugins.append('rarextract')
#file.greenlet = None
#file.on_greenlet_finish()
#file.on_greenlet_stopped()
file.state = 'rarextract_complete'
file.init_progress(1)
file.set_progress(1)
#file.stop()
#path.finished.set()
event.fire('rarextract:part_complete', path, file)
def new_data(self, data):
"""called when new data or new line"""
if "packed data CRC failed in volume" in data:
return self.kill('checksum error in rar archive')
if data.startswith("CRC failed in the encrypted file"): # corrupt file or download not complete
return self.kill('checksum error in rar archive. wrong password?')
if "bad archive" in data.lower():
return self.kill('Bad archive')
m = re.search(r"Insert disk with (.*?) \[C\]ontinue\, \[Q\]uit", data)
if not m:
return
if self.current is not None:
self.finish_file(*self.current)
self.next = m.group(1)
print "setting self.next", self.next
return self.find_next()
def find_next(self):
print "finding next", self.next
next = self.next
if next not in self.parts:
# check if file is in core.files()
found = False
name = os.path.basename(next)
for f in core.files():
if f.name == name and f.get_complete_file() == next:
if not f.working and 'download' in f.completed_plugins:
found = True
current = fileplugin.FilePath(next), f
current[0].finished = AsyncResult()
self.parts[next] = current
print('got next part from idle {}: {}'.format(next, self.current[0]))
break
if f.state == "download":
found = True
break
print "found path but not valid", f.state, f.working
if not found:
# file is not in system, check if it exists on hdd
if os.path.exists(next):
current = fileplugin.FilePath(next), self.first[1]
current[0].finished = AsyncResult()
self.parts[next] = current
print('got next part from hdd {}: {}'.format(next, self.current[0]))
else:
# part does not exists. fail this extract
return self.kill('missing part {}'.format(next))
if next not in self.parts:
print('waiting for part {}'.format(next))
event.fire('rarextract:waiting_for_part', next)
@event.register("file:last_error")
def killit(e, f):
if not f.name == name and f.get_complete_file() == next:
return
if all(f.last_error for f in core.files() if f.name == name and f.get_complete_file() == next):
event.remove("file:last_error", killit)
self.kill('all of the next parts are broken.')
while next not in self.parts:
self.next_part_event.get()
self.next_part_event = AsyncResult()
log.debug('got next part from wait {}: {}'.format(next, self.current[0]))
self.current = self.parts[next]
self.add_library_files()
return self.go_on()
def add_library_files(self):
"""Add extracted files into the library"""
path = fileplugin.FilePath(self.current[0])
f = self.first[1]
with transaction:
if not self.library:
print "Creating package for", path.basename
name = "{} {}".format("Extracted files from", os.path.basename(path.basename))
for p in core.packages():
if p.name == name:
self.library = p
self._library_added = set(f.name for f in p.files)
print "\treused package", p.id
print "package", p.id, p.tab
if not self.library:
self.library = f.package.clone_empty(
name=name,
tab="complete",
state="download_complete",
)
@event.register("package:deleted")
@event.register("file:deleted")
def _deleted_library(event, package):
if event.startswith("file:"):
package = package.package
if package.id == self.library.id:
event.remove(_deleted_library)
for f in self.library.files:
f.delete_local_files()
self.kill("Extracted files have been deleted.", False)
self._deleted_library = _deleted_library
rar = rarfile.RarFile(path, ignore_next_part_missing=True)
print "password is", self.password
try:
if not rar.infolist():
rar.setpassword(self.password)
except rarfile.BadRarFile:
if not rar.infolist():
self.library.delete()
return
links = []
for item in rar.infolist():
name = item.filename
print "From new infolist:", name
if name in self._library_added:
print "\t already added"
continue
elif item.isdir():
print "\t is dir"
continue
else:
self._library_added.add(name)
print "creating file for", repr(name), self.library
links.append(dict(
name=name,
size=item.file_size,
url=u'file://' + os.path.join(
f.get_extract_path().decode(sys.getfilesystemencoding()),
name),
))
if links:
core.add_links(links, package_id=self.library.id)
def go_on(self):
if self.rar is None:
return self.run(*self.current)
if not os.path.exists(self.next):
return
self.rar.stdin.write("C\n")
self.rar.stdin.flush()
if self.current[1] is not None:
with core.transaction:
self.current[1].greenlet = gevent.getcurrent()
self.current[1].greenlet.link(self.current[0].finished)
self.current[1].on_greenlet_started()
self.current[1].log.info("extract go on: {}".format(self.current[1].name))
return True
def kill(self, exc="", _del_lib=True):
blacklist.add(self.first[0].basename) # no autoextract for failed archives
if _del_lib:
self.library.delete()
print "killing rarextract", self.first[0].basename
if isinstance(exc, basestring):
exc = ValueError(exc)
self.current = None
self.killed = True
if self.rar is not None:
self.rar.terminate()
self.rar = None
try:
del extractors[self.id]
except KeyError:
pass
self.next_part_event.set_exception(exc)
for path, file in self.parts.values():
if not path.finished.ready():
path.finished.set_exception(exc)
with transaction:
for path, file in self.parts.values():
if file is not None:
file.stop()
if file.state == 'rarextract_complete':
file.state = 'rarextract'
file.enabled = False
if 'rarextract' in file.completed_plugins:
file.completed_plugins.remove('rarextract')
self.first[1].fatal('rarextract: {}'.format(exc))
return exc
def close(self):
"""called when process is closed"""
if not self.library:
self.add_library_files()
try:
del extractors[self.id]
except KeyError:
pass
if self._deleted_library:
event.remove("package:deleted", self._deleted_library)
if not self.killed:
if self.current is not None:
self.finish_file(*self.current)
if core.config.delete_extracted_archives:
with transaction:
for path, file in self.parts.values():
if file:
file.delete_local_files()
file.fatal('extracted and deleted', type='info', abort_greenlet=False)
else:
os.remove(path)
else:
for path, file in self.parts.values():
if file:
file.log.info('extract complete')
def new_netting_channel(
self,
partner: typing.Address,
settle_timeout: int,
) -> typing.ChannelID:
""" Creates a new channel in the TokenNetwork contract.
Args:
partner: The peer to open the channel with.
settle_timeout: The settle timout to use for this channel.
Returns:
The address of the new netting channel.
"""
if not isaddress(partner):
raise InvalidAddress(
'Expected binary address format for channel partner')
invalid_timeout = (settle_timeout < NETTINGCHANNEL_SETTLE_TIMEOUT_MIN
or
settle_timeout > NETTINGCHANNEL_SETTLE_TIMEOUT_MAX)
if invalid_timeout:
raise InvalidSettleTimeout(
'settle_timeout must be in range [{}, {}]'.format(
NETTINGCHANNEL_SETTLE_TIMEOUT_MIN,
NETTINGCHANNEL_SETTLE_TIMEOUT_MAX))
if self.node_address == partner:
raise SamePeerAddress(
'The other peer must not have the same address as the client.')
# Prevent concurrent attempts to open a channel with the same token and
# partner address.
if partner not in self.open_channel_transactions:
new_open_channel_transaction = AsyncResult()
self.open_channel_transactions[
partner] = new_open_channel_transaction
try:
transaction_hash = self._new_netting_channel(
partner, settle_timeout)
except Exception as e:
new_open_channel_transaction.set_exception(e)
raise
else:
new_open_channel_transaction.set(transaction_hash)
finally:
self.open_channel_transactions.pop(partner, None)
else:
# All other concurrent threads should block on the result of opening this channel
transaction_hash = self.open_channel_transactions[partner].get()
channel_created = self.channel_exists(partner)
if channel_created is False:
log.error('creating new channel failed',
peer1=pex(self.node_address),
peer2=pex(partner))
raise RuntimeError('creating new channel failed')
channel_identifier = self.detail_channel(partner)['channel_identifier']
log.info(
'new_netting_channel called',
peer1=pex(self.node_address),
peer2=pex(partner),
channel_identifier=channel_identifier,
)
return channel_identifier
class TaskExecutor(object):
def __init__(self, balancer, index):
self.balancer = balancer
self.index = index
self.task = None
self.proc = None
self.pid = None
self.conn = None
self.state = WorkerState.STARTING
self.key = str(uuid.uuid4())
self.result = AsyncResult()
self.exiting = False
self.killed = False
self.thread = gevent.spawn(self.executor)
self.cv = Condition()
self.status_lock = RLock()
def checkin(self, conn):
with self.cv:
self.balancer.logger.debug(
'Check-in of worker #{0} (key {1})'.format(
self.index, self.key))
self.conn = conn
self.state = WorkerState.IDLE
self.cv.notify_all()
def put_progress(self, progress):
st = TaskStatus(None)
st.__setstate__(progress)
self.task.set_state(progress=st)
def put_status(self, status):
with self.cv:
# Try to collect rusage at this point, when process is still alive
try:
kinfo = self.balancer.dispatcher.threaded(
bsd.kinfo_getproc, self.pid)
self.task.rusage = kinfo.rusage
except LookupError:
pass
if status['status'] == 'ROLLBACK':
self.task.set_state(TaskState.ROLLBACK)
if status['status'] == 'FINISHED':
self.result.set(status['result'])
if status['status'] == 'FAILED':
error = status['error']
if error['type'] in ERROR_TYPES:
cls = ERROR_TYPES[error['type']]
exc = cls(code=error['code'],
message=error['message'],
stacktrace=error['stacktrace'],
extra=error.get('extra'))
else:
exc = OtherException(
code=error['code'],
message=error['message'],
stacktrace=error['stacktrace'],
type=error['type'],
extra=error.get('extra'),
)
self.result.set_exception(exc)
def put_warning(self, warning):
self.task.add_warning(warning)
def update_env(self, env):
self.conn.call_sync('taskproxy.update_env', env)
def run(self, task):
def match_file(module, f):
name, ext = os.path.splitext(f)
return module == name and ext in ['.py', '.pyc', '.so']
with self.cv:
self.cv.wait_for(lambda: self.state == WorkerState.ASSIGNED)
self.result = AsyncResult()
self.task = task
self.task.set_state(TaskState.EXECUTING)
self.state = WorkerState.EXECUTING
self.cv.notify_all()
self.balancer.logger.debug('Actually starting task {0}'.format(
task.id))
filename = None
module_name = inspect.getmodule(task.clazz).__name__
for dir in self.balancer.dispatcher.plugin_dirs:
found = False
try:
for root, _, files in os.walk(dir):
file = first_or_default(
lambda f: match_file(module_name, f), files)
if file:
filename = os.path.join(root, file)
found = True
break
if found:
break
except OSError:
continue
try:
self.conn.call_sync(
'taskproxy.run', {
'id': task.id,
'user': task.user,
'class': task.clazz.__name__,
'filename': filename,
'args': task.args,
'debugger': task.debugger,
'environment': task.environment,
'hooks': task.hooks,
})
except RpcException as e:
self.balancer.logger.warning(
'Cannot start task {0} on executor #{1}: {2}'.format(
task.id, self.index, str(e)))
self.balancer.logger.warning(
'Killing unresponsive task executor #{0} (pid {1})'.format(
self.index, self.proc.pid))
self.terminate()
try:
self.result.get()
except BaseException as e:
if isinstance(e, OtherException):
self.balancer.dispatcher.report_error(
'Task {0} raised invalid exception'.format(self.task.name),
e)
if isinstance(e, TaskAbortException):
self.task.set_state(TaskState.ABORTED,
TaskStatus(0, 'aborted'))
else:
self.task.error = serialize_error(e)
self.task.set_state(
TaskState.FAILED,
TaskStatus(0,
str(e),
extra={"stacktrace": traceback.format_exc()}))
with self.cv:
self.task.ended.set()
if self.state == WorkerState.EXECUTING:
self.state = WorkerState.IDLE
self.cv.notify_all()
self.balancer.task_exited(self.task)
return
with self.cv:
self.task.result = self.result.value
self.task.set_state(TaskState.FINISHED, TaskStatus(100, ''))
self.task.ended.set()
if self.state == WorkerState.EXECUTING:
self.state = WorkerState.IDLE
self.cv.notify_all()
self.balancer.task_exited(self.task)
def abort(self):
self.balancer.logger.info("Trying to abort task #{0}".format(
self.task.id))
# Try to abort via RPC. If this fails, kill process
try:
# If task supports abort protocol we don't need to worry about subtasks - it's task
# responsibility to kill them
self.conn.call_sync('taskproxy.abort')
except RpcException as err:
self.balancer.logger.warning(
"Failed to abort task #{0} gracefully: {1}".format(
self.task.id, str(err)))
self.balancer.logger.warning("Killing process {0}".format(
self.pid))
self.killed = True
self.terminate()
# Now kill all the subtasks
for subtask in filter(lambda t: t.parent is self.task,
self.balancer.task_list):
self.balancer.logger.warning(
"Aborting subtask {0} because parent task {1} died".format(
subtask.id, self.task.id))
self.balancer.abort(subtask.id)
def terminate(self):
try:
self.proc.terminate()
except OSError:
self.balancer.logger.warning(
'Executor process with PID {0} already dead'.format(
self.proc.pid))
def executor(self):
while not self.exiting:
try:
self.proc = Popen([TASKWORKER_PATH, self.key,
str(self.index)],
close_fds=True,
preexec_fn=os.setpgrp,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
self.pid = self.proc.pid
self.balancer.logger.debug(
'Started executor #{0} as PID {1}'.format(
self.index, self.pid))
except OSError:
self.result.set_exception(
TaskException(errno.EFAULT, 'Cannot spawn task executor'))
self.balancer.logger.error(
'Cannot spawn task executor #{0}'.format(self.index))
return
for line in self.proc.stdout:
line = line.decode('utf8')
self.balancer.logger.debug('Executor #{0}: {1}'.format(
self.index, line.strip()))
if self.task:
self.task.output += line
self.proc.wait()
with self.cv:
self.state = WorkerState.STARTING
self.cv.notify_all()
if self.proc.returncode == -signal.SIGTERM:
self.balancer.logger.info(
'Executor process with PID {0} was terminated gracefully'.
format(self.proc.pid))
else:
self.balancer.logger.error(
'Executor process with PID {0} died abruptly with exit code {1}'
.format(self.proc.pid, self.proc.returncode))
if self.killed:
self.result.set_exception(
TaskException(errno.EFAULT, 'Task killed'))
else:
self.result.set_exception(
TaskException(errno.EFAULT, 'Task executor died'))
gevent.sleep(1)
def die(self):
self.exiting = True
if self.proc:
self.terminate()
class InputTable(scheme.Table):
_table_name = 'input'
_table_collection = input_tables
id = scheme.Column('api')
type = scheme.Column('api')
parent = scheme.Column(
'api',
lambda self, value: value and [value._table_name, value.id] or None)
timeout = scheme.Column(
'api',
lambda self, timeout: timeout and int(timeout.eta * 1000) or None)
elements = scheme.Column('api')
result = scheme.Column('api')
close_aborts = scheme.Column('api')
ignore_api = False
def __init__(self,
type,
parent,
timeout,
elements,
close_aborts,
ignore_api=False):
self.type = type
self.parent = parent
self.timeout = None
self.elements = [isinstance(e, list) and e or [e] for e in elements]
self.close_aborts = close_aborts
self.ignore_api = ignore_api
if parent:
parent.input = self
self._result = AsyncResult()
self.reset_timeout(timeout)
def set_result(self, value):
if self._result.ready():
#raise RuntimeError('result of input already set')
return
with scheme.transaction:
self.result = value
self.reset_timeout(None)
self._result.set(value)
event.fire("input:result", self)
def set_error(self, value):
if self._result.ready():
#raise RuntimeError('result of input already set')
return
with scheme.transaction:
self.result = str(value)
self.reset_timeout(None)
self._result.set_exception(value)
event.fire("input:error", self)
def reset_timeout(self, timeout):
with scheme.transaction:
if self.timeout:
self.timeout.kill()
if timeout:
self.timeout = gevent.spawn_later(timeout, self._timed_out)
self.timeout.eta = time.time() + timeout
elif self.timeout:
self.timeout = None
def _timed_out(self):
with scheme.transaction:
self.timeout = None
self.set_error(InputTimeout())
def new_netting_channel(self, other_peer: Address,
settle_timeout: int) -> Address:
""" Creates and deploys a new netting channel contract.
Args:
other_peer: The peer to open the channel with.
settle_timeout: The settle timout to use for this channel.
Returns:
The address of the new netting channel.
"""
if not is_binary_address(other_peer):
raise ValueError('The other_peer must be a valid address')
invalid_timeout = (settle_timeout < NETTINGCHANNEL_SETTLE_TIMEOUT_MIN
or
settle_timeout > NETTINGCHANNEL_SETTLE_TIMEOUT_MAX)
if invalid_timeout:
raise InvalidSettleTimeout(
'settle_timeout must be in range [{}, {}]'.format(
NETTINGCHANNEL_SETTLE_TIMEOUT_MIN,
NETTINGCHANNEL_SETTLE_TIMEOUT_MAX,
))
local_address = privatekey_to_address(self.client.privkey)
if local_address == other_peer:
raise SamePeerAddress(
'The other peer must not have the same address as the client.')
# Prevent concurrent attempts to open a channel with the same token and
# partner address.
if other_peer not in self.open_channel_transactions:
new_open_channel_transaction = AsyncResult()
self.open_channel_transactions[
other_peer] = new_open_channel_transaction
try:
transaction_hash = self._new_netting_channel(
other_peer, settle_timeout)
except Exception as e:
new_open_channel_transaction.set_exception(e)
raise
else:
new_open_channel_transaction.set(transaction_hash)
finally:
self.open_channel_transactions.pop(other_peer, None)
else:
# All other concurrent threads should block on the result of opening this channel
self.open_channel_transactions[other_peer].get()
netting_channel_results_encoded = self.proxy.contract.functions.getChannelWith(
to_checksum_address(other_peer), ).call(
{'from': to_checksum_address(self.client.sender)})
# address is at index 0
netting_channel_address_encoded = netting_channel_results_encoded
if not netting_channel_address_encoded:
log.error(
'netting_channel_address failed',
peer1=pex(local_address),
peer2=pex(other_peer),
)
raise RuntimeError('netting_channel_address failed')
netting_channel_address_bin = to_canonical_address(
netting_channel_address_encoded)
log.info(
'new_netting_channel called',
peer1=pex(local_address),
peer2=pex(other_peer),
netting_channel=pex(netting_channel_address_bin),
)
return netting_channel_address_bin
class AceClient(object):
def __init__(self, acehost, aceAPIport, aceHTTPport, acehostslist, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = AceConfig.videoseekback
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0;
# Logger
logger = logging.getLogger('AceClientimport tracebacknt_init')
# Try to connect AceStream engine
try:
self._socket = telnetlib.Telnet(acehost, aceAPIport, connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = acehost, aceAPIport, aceHTTPport
logger.debug("Successfully connected to AceStream on %s:%d" % (acehost, aceAPIport))
except:
for AceEngine in acehostslist:
try:
self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[0], AceEngine[1], AceEngine[2]
logger.debug("Successfully connected to AceStream on %s:%d" % (AceEngine[0], AceEngine[1]))
break
except:
logger.debug("The are no alive AceStream on %s:%d" % (AceEngine[0], AceEngine[1]))
pass
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger("AceClient_write")
logger.debug(message)
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException("Result not received")
except gevent.Timeout:
raise AceException("Timeout")
return result
def START(self, datatype, value):
'''
Start video method
'''
if self._engine_version_code >= 3010500 and AceConfig.vlcuse:
stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \
+ ' transcode_mp3=' + str(AceConfig.transcode_mp3) \
+ ' transcode_ac3=' + str(AceConfig.transcode_ac3) \
+ ' preferred_audio_language=' + AceConfig.preferred_audio_language
else:
stream_type = 'output_format=http'
self._urlresult = AsyncResult()
self._write(AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state is not None and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, value):
self._result = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), random.randint(1, AceConfig.maxconns * 10000), value))
return self._getResult()
def GETCONTENTINFO(self, datatype, value):
dict = {'torrent': 'url', 'infohash':'infohash', 'raw':'data', 'pid':'content_id'}
self.paramsdict={dict[datatype]:value,'developer_id':'0','affiliate_id':'0','zone_id':'0'}
contentinfo = self.LOADASYNC(datatype, self.paramsdict)
return contentinfo
def GETCID(self, datatype, url):
contentinfo = self.GETCONTENTINFO(datatype, url)
self._cidresult = AsyncResult()
self._write(AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def getUrl(self, timeout=30):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "Engine response time exceeded. GetURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger("StreamReader")
logger.debug("Opening video stream: %s" % url)
logger.debug("Get headers from client: %s" % req_headers)
self._streamReaderState = 1
try:
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return None
# Sending client hedaers to AceEngine
if req_headers.has_key('range'):
del req_headers['range']
connection = self._streamReaderConnection = requests.get(url, headers=req_headers, stream=True)
if connection.status_code not in (200, 206):
logger.error("Failed to open video stream %s" % url)
return None
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
while True:
data = None
clients = counter.getClients(cid)
try:
data = connection.raw.read(AceConfig.readchunksize)
except:
break;
if data and clients:
with self._lock:
if len(self._streamReaderQueue) == AceConfig.readchunksize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" % str(c))
c.destroy()
elif not clients:
logger.debug("All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except requests.exceptions.ConnectionError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClients(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger("StreamReader")
c = self._streamReaderConnection
if c:
self._streamReaderConnection = None
c.close()
logger.debug("Video stream closed")
self._streamReaderQueue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'version_code=' in self._recvbuffer:
v = self._recvbuffer.find('version_code=')
self._engine_version_code = int(self._recvbuffer[v+13:v+20])
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except requests.exceptions.ConnectionError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
self._result.set(_contentinfo)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or self._started_again or not self._recvbuffer.endswith(' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS)
# STOP
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
# SHUTDOWN
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
# AUTH
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
# GETUSERDATA
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
# LIVEPOS
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
if self._seekback and not self._started_again:
self._write(AceMessage.request.LIVESEEK(str(int(self._position_last) - self._seekback)))
logger.debug('Seeking back')
self._started_again = True
# DOWNLOADSTOP
elif self._recvbuffer.startswith(AceMessage.response.DOWNLOADSTOP):
self._state = self._recvbuffer.split()[1]
# STATE
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
# STATUS
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._status == 'main:idle':
self._result.set(True)
# PAUSE
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
# RESUME
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep() # PAUSE/RESUME delay
self._resumeevent.set()
# CID
elif self._recvbuffer.startswith('##') or len(self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
logger.debug("CID: %s" %self._recvbuffer)
class AceClient(object):
def __init__(self, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Logger
logger = logging.getLogger('AceClient_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException(
"Socket creation error! Ace is not running? " + repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def _write(self, message):
try:
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
logger = logging.getLogger("AceClient_START")
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
errmsg = "START error!"
logger.error(errmsg)
raise AceException(errmsg)
except gevent.Timeout:
errmsg = "START timeout!"
logger.error(errmsg)
raise AceException(errmsg)
return result
def START(self, datatype, value):
'''
Start video method
'''
self._result = AsyncResult()
self._urlresult = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value))
contentinfo = self._getResult()
self._write(AceMessage.request.START(datatype.upper(), value))
self._getResult()
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
self._resumeevent.wait(timeout=timeout)
return
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin+4:self._request_key_begin+14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except urllib2.URLError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
_filename = urllib2.unquote(_contentinfo.get('files')[0][0])
self._result.set(_filename)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
class StreamingExtract(object):
def __init__(self, id, hddsem, threadpool):
self.id = id
self.hddsem = hddsem
self.threadpool = threadpool
self.password = None
self.killed = False
self.parts = dict()
self.first = None
self.current = None
self.next = None
self.next_part_event = AsyncResult()
self.rar = None
self.library = None
self._library_added = set()
self._deleted_library = None
extractors[id] = self
def feed_part(self, path, file):
path.finished = AsyncResult()
self.parts[path.path] = path, file
log.debug('fed new part {}: {}'.format(path, path))
if file.state != 'rarextract':
with transaction:
file.state = 'rarextract'
if self.first is None:
self.first = self.current = path, file
self.add_library_files()
self.run(path, file)
else:
if path.path == self.next:
self.next_part_event.set(path)
path.finished.get()
def run(self, path, file):
try:
self.first = self.current
with transaction:
file.greenlet = gevent.getcurrent()
file.on_greenlet_started()
try:
result = self.bruteforce(path, file)
except rarfile.NeedFirstVolume:
self.next = os.path.join(path.dir, "{}.part{}.rar".format(path.basename, "1".zfill(len(path.part))))
self.find_next()
if core.config.delete_extracted_archives:
return False
return
if result and result is not True:
raise result
if self.password:
rarpw = "-p"+self.password
else:
rarpw = "-p-"
cmd = [rarfile.UNRAR_TOOL, "x", "-y", rarpw, "-idq", "-vp", path, file.get_extract_path() + os.sep]
file.log.info("starting extraction of {} with params {}".format(path[1:], cmd))
self.rar = rarfile.custom_popen(cmd)
self.wait_data()
if not path.finished.ready():
path.finished.set()
if core.config.delete_extracted_archives:
return False
except BaseException as e:
traceback.print_exc()
self.kill(e)
raise
def bruteforce(self, path, file):
rar = rarfile.RarFile(path, ignore_next_part_missing=True)
if rar.not_first_volume:
raise rarfile.NeedFirstVolume("First Volume for extraction")
if not rar.needs_password():
self.password = None
return
passwords = []
for i in itertools.chain(file.package.extract_passwords, core.config.bruteforce_passwords):
if not i in passwords:
passwords.append(i)
if rar.needs_password() and rar.infolist():
pw = bruteforce_by_content(rar, passwords)
if not pw:
print "could not find password, asking user"
for pw in file.solve_password(
message="Rarfile {} password cannot be cracked. Enter correct password: #".format(path.name),
retries=5):
pw = bruteforce_by_content(rar, [pw])
if pw:
break
else:
return self.kill('extract password not entered')
else:
print "Found password by content:", pw
self.password = pw
return
print "testing", passwords
if not self.threadpool.apply(bruteforce, (rar, passwords, self.hddsem, file.log)):
# ask user for password
for pw in file.solve_password(
message="Enter the extract password for file: {} #".format(path.name),
retries=5):
if self.threadpool.apply(bruteforce, (rar, [pw], self.hddsem, file.log)):
break
else:
return self.kill('extract password not entered')
self.password = rar._password
if self.password and self.password not in core.config.bruteforce_passwords:
with transaction:
core.config.bruteforce_passwords.append(self.password)
def wait_data(self):
bytes = ''
while True:
data = self.rar.stdout.read(1)
if not data:
break
bytes += data
for i in bytes.splitlines():
if i:
result = self.new_data(i)
if result is True:
bytes = ''
if result and result is not True:
raise result
self.close()
def finish_file(self, path, file):
if file is not None:
with core.transaction:
#if not 'rarextract' in file.completed_plugins:
# file.completed_plugins.append('rarextract')
#file.greenlet = None
#file.on_greenlet_finish()
#file.on_greenlet_stopped()
file.state = 'rarextract_complete'
file.init_progress(1)
file.set_progress(1)
#file.stop()
#path.finished.set()
event.fire('rarextract:part_complete', path, file)
def new_data(self, data):
"""called when new data or new line"""
if "packed data CRC failed in volume" in data:
return self.kill('checksum error in rar archive')
if data.startswith("CRC failed in the encrypted file"): # corrupt file or download not complete
return self.kill('checksum error in rar archive. wrong password?')
if "bad archive" in data.lower():
return self.kill('Bad archive')
m = re.search(r"Insert disk with (.*?((\.part\d+)?\.r..)) \[C\]ontinue\, \[Q\]uit", data)
if not m:
return
if self.current is not None:
self.finish_file(*self.current)
self.next = self.first[0].basename + m.group(2)
print "setting self.next", self.next
return self.find_next()
def find_next(self):
print "finding next", self.next
next = self.next
if next not in self.parts:
# check if file is in core.files()
found = False
name = os.path.basename(next)
for f in core.files():
if f.name == name and f.get_complete_file() == next:
if not f.working and 'download' in f.completed_plugins:
found = True
current = fileplugin.FilePath(next), f
current[0].finished = AsyncResult()
self.parts[next] = current
print('got next part from idle {}: {}'.format(next, self.current[0]))
break
if f.state == "download":
found = True
break
print "found path but not valid", f.state, f.working
if not found:
# file is not in system, check if it exists on hdd
if os.path.exists(next):
current = fileplugin.FilePath(next), self.first[1]
current[0].finished = AsyncResult()
self.parts[next] = current
print('got next part from hdd {}: {}'.format(next, self.current[0]))
else:
# part does not exists. fail this extract
return self.kill('missing part {}'.format(next))
if next not in self.parts:
print('waiting for part {}'.format(next))
event.fire('rarextract:waiting_for_part', next)
@event.register("file:last_error")
def killit(e, f):
if not f.name == name and f.get_complete_file() == next:
return
if all(f.last_error for f in core.files() if f.name == name and f.get_complete_file() == next):
event.remove("file:last_error", killit)
self.kill('all of the next parts are broken.')
while next not in self.parts:
self.next_part_event.get()
self.next_part_event = AsyncResult()
log.debug('got next part from wait {}: {}'.format(next, self.current[0]))
self.current = self.parts[next]
self.add_library_files()
return self.go_on()
def add_library_files(self):
"""Add extracted files into the library"""
path = fileplugin.FilePath(self.current[0])
f = self.first[1]
with transaction:
if not self.library:
print "Creating package for", path.basename
name = "{} {}".format("Extracted files from", os.path.basename(path.basename))
for p in core.packages():
if p.name == name:
self.library = p
self._library_added = set(f.name for f in p.files)
print "\treused package", p.id
print "package", p.id, p.tab
if not self.library:
self.library = f.package.clone_empty(
name=name,
tab="complete",
state="download_complete",
)
@event.register("package:deleted")
@event.register("file:deleted")
def _deleted_library(e, package):
import traceback
print traceback.print_stack()
print "---------", e
if e.startswith("file:"):
package = package.package
if package.id == self.library.id:
event.remove("package:deleted", _deleted_library)
event.remove("file:deleted", _deleted_library)
for f in self.library.files:
f.delete_local_files()
self.kill("Extracted files have been deleted.", False)
self._deleted_library = _deleted_library
rar = rarfile.RarFile(path, ignore_next_part_missing=True)
print "password is", self.password
try:
if not rar.infolist():
rar.setpassword(self.password)
except rarfile.BadRarFile:
if not rar.infolist():
self.library.delete()
return
links = []
for item in rar.infolist():
name = item.filename
print "From new infolist:", name
if name in self._library_added:
print "\t already added"
continue
elif item.isdir():
print "\t is dir"
continue
else:
self._library_added.add(name)
print "creating file for", repr(name), self.library
links.append(dict(
name=name,
size=item.file_size,
url=u'file://' + os.path.join(
f.get_extract_path().decode(sys.getfilesystemencoding()),
name),
))
if links:
core.add_links(links, package_id=self.library.id)
def go_on(self):
if self.rar is None:
return self.run(*self.current)
if not os.path.exists(self.next):
return
self.rar.stdin.write("C\n")
self.rar.stdin.flush()
if self.current[1] is not None:
with core.transaction:
self.current[1].greenlet = gevent.getcurrent()
self.current[1].greenlet.link(self.current[0].finished)
self.current[1].on_greenlet_started()
self.current[1].log.info("extract go on: {}".format(self.current[1].name))
return True
def kill(self, exc="", _del_lib=True):
if self.killed:
return exc
self.killed = True
blacklist.add(self.first[0].basename) # no autoextract for failed archives
if _del_lib:
self.library.delete()
print "killing rarextract", self.first[0].basename, exc
if isinstance(exc, basestring):
exc = ValueError(exc)
self.current = None
if self.rar is not None:
self.rar.terminate()
self.rar = None
try:
del extractors[self.id]
except KeyError:
pass
self.next_part_event.set_exception(exc)
for path, file in self.parts.values():
if not path.finished.ready():
path.finished.set_exception(exc)
with transaction:
for path, file in self.parts.values():
if file is not None:
file.stop()
if file.state == 'rarextract_complete':
file.state = 'rarextract'
file.enabled = False
if 'rarextract' in file.completed_plugins:
file.completed_plugins.remove('rarextract')
self.first[1].fatal('rarextract: {}'.format(exc))
return exc
def close(self):
"""called when process is closed"""
if not self.library:
self.add_library_files()
try:
del extractors[self.id]
except KeyError:
pass
if self._deleted_library:
event.remove("package:deleted", self._deleted_library)
if not self.killed:
if self.current is not None:
self.finish_file(*self.current)
if core.config.delete_extracted_archives:
with transaction:
for path, file in self.parts.values():
if file:
file.delete_local_files()
file.fatal('extracted and deleted', type='info', abort_greenlet=False)
else:
os.remove(path)
else:
for path, file in self.parts.values():
if file:
file.log.info('extract complete')
class AceClient(object):
def __init__(self, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = 0
# Did we get START command again? For seekback.
self._started_again = False
# Logger
logger = logging.getLogger('AceClient_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException(
"Socket creation error! Ace is not running? " + repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def _write(self, message):
try:
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0, seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
logger = logging.getLogger("AceClient_START")
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
errmsg = "START error!"
logger.error(errmsg)
raise AceException(errmsg)
except gevent.Timeout:
errmsg = "START timeout!"
logger.error(errmsg)
raise AceException(errmsg)
return result
def START(self, datatype, value):
'''
Start video method
'''
self._result = AsyncResult()
self._urlresult = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value))
contentinfo = self._getResult()
self._write(AceMessage.request.START(datatype.upper(), value))
self._getResult()
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
#logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin+4:self._request_key_begin+14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except:
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
_filename = urllib2.unquote(_contentinfo.get('files')[0][0])
self._result.set(_filename)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or (self._seekback and self._started_again):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seeback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
# logger.debug('Current position/last/buf: %s/%s/%s' % (self._position, self._position_last, self._position_buf))
if self._seekback and not self._started_again:
self._write(AceMessage.request.SEEK(str(int(self._position_last) - \
self._seekback)))
logger.debug('Seeking back')
self._started_again = True
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to {0} data {1}".format(self._status, repr(self._recvbuffer)))
if self._status == 'main:dl':
logger.debug("progress - speed {3}kb/s peers {6}".format(*self._recvbuffer.split(';')))
elif self._status == 'main:prebuf' or self._status == 'main:buf':
logger.debug("progress {1}% speed {5}kb/s peers {8}".format(*self._recvbuffer.split(';')))
elif self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
class ChildrenMonitor(object):
"""Simple monitor that monitors the children of a node and their
content.
"""
_STOP_REQUEST = object()
def __init__(self, client, path, into, factory, args, listener):
self.client = client
self.path = path
self.into = into if into is not None else {}
self.factory = factory if factory is not None else str
self.args = args
self.listener = listener or MonitorListener()
self.started = AsyncResult()
self.queue = Queue()
self.stats = {}
self._delay = 1.343
self.max_delay = 180
def _monitor(self):
"""Run the monitoring loop."""
def watcher(event):
self.queue.put(event)
while True:
try:
children = self.client.get_children(self.path, watcher)
except zookeeper.NoNodeException:
if not self.started.ready():
self.started.set(None)
gevent.sleep(1)
continue
except (zookeeper.ConnectionLossException,
zookeeper.SessionExpiredException,
zookeeper.InvalidStateException) as err:
if not self.started.ready():
self.started.set_exception(err)
break
logging.error("got %r while monitoring %s", str(err),
self.path)
gevent.sleep(self._delay)
self._delay += self._delay * random.random()
self._delay = min(self._delay, self.max_delay)
continue
except Exception, err:
if not self.started.ready():
self.started.set_exception(err)
break
raise
for child in children:
if not child in self.stats:
try:
data, stat = self.client.get(os.path.join(self.path,
child))
except zookeeper.NoNodeException:
print "race condition while getting", os.path.join(
self.path, child)
else:
self.into[child] = self.factory(data, *self.args)
self.listener.created(child, self.into[child])
self.stats[child] = stat
else:
try:
data, stat = self.client.get(os.path.join(self.path,
child))
except zookeeper.NoNodeException:
print "race condition while getting", os.path.join(
self.path, child)
# should we remove it here?
else:
if stat['version'] != self.stats[child]['version']:
self.into[child] = self.factory(data, *self.args)
self.listener.modified(child, self.into[child])
self.stats[child] = stat
for child in self.into.keys():
if child not in children:
del self.into[child]
del self.stats[child]
self.listener.deleted(child)
if not self.started.ready():
self.started.set(None)
self.listener.commit()
event = self.queue.get()
if event is self._STOP_REQUEST:
break
class AceClient(object):
def __init__(self, ace, connect_timeout=5, result_timeout=10):
# Telnet response buffer
self._recvbuffer = None
# AceEngine socket
self._socket = None
# AceEngine read result timeout
self._resulttimeout = float(result_timeout)
# Shutting down flag
self._shuttingDown = Event()
# AceEngine product key
self._product_key = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
# Result for START URL
self._urlresult = AsyncResult()
# URL response time from AceEngine
self._videotimeout = None
# Result for CID
self._cidresult = AsyncResult()
# Current STATUS
self._status = AsyncResult()
# Current EVENT
self._event = AsyncResult()
# Current STATE
self._state = AsyncResult()
# Current AUTH
self._gender = None
self._age = None
# Seekback seconds.
self._seekback = None
# Did we get START command again? For seekback.
self._started_again = Event()
# AceEngine Streamreader ring buffer with max number of chunks in queue
self._streamReaderQueue = gevent.queue.Queue(maxsize=1000)
# Logger
logger = logging.getLogger('AceClient')
# Try to connect AceStream engine
try:
self._socket = Telnet(ace['aceHostIP'], ace['aceAPIport'], connect_timeout)
logger.debug('Successfully connected to AceStream on %s:%s' % (ace['aceHostIP'], ace['aceAPIport']))
except:
errmsg = 'The are no alive AceStream Engines found!'
raise AceException(errmsg)
else:
# Spawning telnet data reader greenlet
gevent.spawn(self._recvData)
def destroy(self):
'''
AceClient Destructor
'''
logger = logging.getLogger('AceClient_destroy') # Logger
if self._shuttingDown.ready(): return # Already in the middle of destroying
self._result.set()
# Trying to disconnect
try:
logger.debug('Destroying AceStream client.....')
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except: pass # Ignore exceptions on destroy
finally: self._shuttingDown.set()
def reset(self):
self._started_again.clear()
self._result.set()
self._urlresult.set()
def _write(self, message):
try:
logger = logging.getLogger('AceClient_write')
logger.debug('>>> %s' % message)
self._socket.write('%s\r\n' % message)
except EOFError as e: raise AceException('Write error! %s' % repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_25_34, product_key=None, videoseekback=0, videotimeout=0):
self._gender = gender
self._age = age
self._product_key = product_key
self._seekback = videoseekback
self._videotimeout = float(videotimeout)
self._started_again.clear()
logger = logging.getLogger('AceClient_aceInit')
self._result = AsyncResult()
self._write(AceMessage.request.HELLO) # Sending HELLOBG
try: params = self._result.get(timeout=self._resulttimeout)
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. HELLOTS not resived!' % self._resulttimeout
raise AceException(errmsg)
return
self._result = AsyncResult()
self._write(AceMessage.request.READY(params.get('key',''), self._product_key))
try:
if self._result.get(timeout=self._resulttimeout) == 'NOTREADY': # Get NOTREADY instead AUTH user_auth_level
errmsg = 'NOTREADY recived from AceEngine! Wrong acekey?'
raise AceException(errmsg)
return
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. AUTH not resived!' % self._resulttimeout
raise AceException(errmsg)
if int(params.get('version_code', 0)) >= 3003600: # Display download_stopped massage
params_dict = {'use_stop_notifications': '1'}
self._write(AceMessage.request.SETOPTIONS(params_dict))
def START(self, command, paramsdict, acestreamtype):
'''
Start video method
Returns the url provided by AceEngine
'''
paramsdict['stream_type'] = ' '.join(['{}={}'.format(k,v) for k,v in acestreamtype.items()])
self._urlresult = AsyncResult()
self._write(AceMessage.request.START(command.upper(), paramsdict))
try: return self._urlresult.get(timeout=self._videotimeout) # Get url for play from AceEngine
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. START URL not resived!' % self._videotimeout
raise AceException(errmsg)
def STOP(self):
'''
Stop video method
'''
self._state = AsyncResult()
self._write(AceMessage.request.STOP)
try: self._state.get(timeout=self._resulttimeout); self._started_again.clear()
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. STATE 0 (IDLE) not resived!' % self._resulttimeout
raise AceException(errmsg)
def LOADASYNC(self, command, params):
self._result = AsyncResult()
self._write(AceMessage.request.LOADASYNC(command.upper(), random.randint(1, 100000), params))
try: return self._result.get(timeout=self._resulttimeout) # Get _contentinfo json
except gevent.Timeout:
errmsg = 'Engine response %ssec time exceeded. LOADARESP not resived!' % self._resulttimeout
raise AceException(errmsg)
def GETCONTENTINFO(self, command, value):
paramsdict = { command:value, 'developer_id':'0', 'affiliate_id':'0', 'zone_id':'0' }
return self.LOADASYNC(command, paramsdict)
def GETCID(self, command, value):
contentinfo = self.GETCONTENTINFO(command, value)
if contentinfo['status'] in (1, 2):
paramsdict = {'checksum':contentinfo['checksum'], 'infohash':contentinfo['infohash'], 'developer_id':'0', 'affiliate_id':'0', 'zone_id':'0'}
self._cidresult = AsyncResult()
self._write(AceMessage.request.GETCID(paramsdict))
try:
cid = self._cidresult.get(timeout=self._resulttimeout)
return '' if cid is None or cid == '' else cid[2:]
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. CID not resived!' % self._resulttimeout
raise AceException(errmsg)
else:
cid = None
errmsg = 'LOADASYNC returned error with message: %s' % contentinfo['message']
raise AceException(errmsg)
def GETINFOHASH(self, command, value, idx=0):
contentinfo = self.GETCONTENTINFO(command, value)
if contentinfo['status'] in (1, 2):
return contentinfo['infohash'], [x[0] for x in contentinfo['files'] if x[1] == int(idx)][0]
elif contentinfo['status'] == 0:
errmsg = 'LOADASYNC returned status 0: The transport file does not contain audio/video files'
raise AceException(errmsg)
else:
errmsg = 'LOADASYNC returned error with message: %s' % contentinfo['message']
raise AceException(errmsg)
def StreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger('StreamReader')
logger.debug('Start StreamReader for url: %s' % url)
self._write(AceMessage.request.EVENT('play'))
with requests.Session() as session:
if req_headers:
logger.debug('Sending headers from client to AceEngine: %s' % req_headers)
session.headers.update(req_headers)
try:
# AceEngine return link for HLS stream
if url.endswith('.m3u8'):
_used_chunks = []
while self._state.get(timeout=self._resulttimeout)[0] in ('2', '3'):
for line in session.get(url, stream=True, timeout = (5,None)).iter_lines():
if self._state.get(timeout=self._resulttimeout)[0] not in ('2', '3'): return
if line.startswith(b'http://') and line not in _used_chunks:
self.RAWDataReader(session.get(line, stream=True, timeout=(5,None)).raw, cid, counter)
_used_chunks.append(line)
if len(_used_chunks) > 15: _used_chunks.pop(0)
gevent.sleep(4)
# AceStream return link for HTTP stream
else: self.RAWDataReader(session.get(url, stream=True, timeout = (5,None)).raw, cid, counter)
except requests.exceptions.HTTPError as err:
logger.error('An http error occurred while connecting to aceengine: %s' % repr(err))
except requests.exceptions.RequestException:
logger.error('There was an ambiguous exception that occurred while handling request')
except Exception as err:
logger.error('Unexpected error in streamreader %s' % repr(err))
finally:
_used_chunks = None
self._streamReaderQueue.queue.clear()
counter.deleteAll(cid)
def RAWDataReader(self, stream, cid, counter):
logger = logging.getLogger('RAWDataReader')
while self._state.get(timeout=self._resulttimeout)[0] in ('2', '3'):
gevent.sleep()
if self._state.get(timeout=self._resulttimeout)[0] == '2': # Read data from AceEngine only if STATE 2 (DOWNLOADING)
data = stream.read(requests.models.CONTENT_CHUNK_SIZE)
if not data: return
try: self._streamReaderQueue.put_nowait(data)
except gevent.queue.Full: self._streamReaderQueue.get_nowait(); self._streamReaderQueue.put_nowait(data)
clients = counter.getClients(cid)
if not clients: return
for c in clients:
try: c.queue.put(data, timeout=5)
except gevent.queue.Full:
if len(clients) > 1:
logger.warning('Client %s does not read data from buffer until 5sec - disconnect it' % c.handler.clientip)
c.destroy()
elif (time.time() - self._state.get(timeout=self._resulttimeout)[1]) >= self._videotimeout: # STATE 3 (BUFFERING)
logger.warning('No data received from AceEngine for %ssec - broadcast stoped' % self._videotimeout); return
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while 1:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until('\r\n').strip()
logger.debug('<<< %s' % requests.compat.unquote(self._recvbuffer))
except gevent.GreenletExit: break
except:
# If something happened during read, abandon reader.
logger.error('Exception at socket read. AceClient destroyed')
if not self._shuttingDown.ready(): self._shuttingDown.set()
return
else: # Parsing everything only if the string is not empty
# HELLOTS
if self._recvbuffer.startswith('HELLOTS'):
# version=engine_version version_code=version_code key=request_key http_port=http_port
self._result.set({ k:v for k,v in (x.split('=') for x in self._recvbuffer.split() if '=' in x) })
# NOTREADY
elif self._recvbuffer.startswith('NOTREADY'): self._result.set('NOTREADY')
# AUTH
elif self._recvbuffer.startswith('AUTH'): self._result.set(self._recvbuffer.split()[1]) # user_auth_level
# START
elif self._recvbuffer.startswith('START'):
# url [ad=1 [interruptable=1]] [stream=1] [pos=position]
params = { k:v for k,v in (x.split('=') for x in self._recvbuffer.split() if '=' in x) }
if not self._seekback or self._started_again.ready() or params.get('stream','') is not '1':
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
self._urlresult.set(self._recvbuffer.split()[1]) # url for play
# LOADRESP
elif self._recvbuffer.startswith('LOADRESP'):
self._result.set(requests.compat.json.loads(requests.compat.unquote(''.join(self._recvbuffer.split()[2:]))))
# STATE
elif self._recvbuffer.startswith('STATE'): # tuple of (state_id, time of appearance)
self._state.set((self._recvbuffer.split()[1], time.time()))
# STATUS
elif self._recvbuffer.startswith('STATUS'):
self._tempstatus = self._recvbuffer.split()[1]
if self._tempstatus.startswith('main:idle'): pass
elif self._tempstatus.startswith('main:loading'): pass
elif self._tempstatus.startswith('main:starting'): pass
elif self._tempstatus.startswith('main:check'): pass
elif self._tempstatus.startswith('main:wait'): pass
elif self._tempstatus.startswith(('main:prebuf','main:buf')): pass #progress;time
#values = list(map(int, self._tempstatus.split(';')[3:]))
#self._status.set({k: v for k, v in zip(AceConst.STATUS, values)})
elif self._tempstatus.startswith('main:dl'): pass
#values = list(map(int, self._tempstatus.split(';')[1:]))
#self._status.set({k: v for k, v in zip(AceConst.STATUS, values)})
elif self._tempstatus.startswith('main:err'): # err;error_id;error_message
self._status.set_exception(AceException('%s with message %s' % (self._tempstatus.split(';')[0],self._tempstatus.split(';')[2])))
# CID
elif self._recvbuffer.startswith('##'): self._cidresult.set(self._recvbuffer)
# INFO
elif self._recvbuffer.startswith('INFO'): pass
# EVENT
elif self._recvbuffer.startswith('EVENT'):
self._tempevent = self._recvbuffer.split()
if self._seekback and not self._started_again.ready() and 'livepos' in self._tempevent:
params = { k:v for k,v in (x.split('=') for x in self._tempevent if '=' in x) }
self._write(AceMessage.request.LIVESEEK(int(params['last']) - self._seekback))
self._started_again.set()
elif 'getuserdata' in self._tempevent: self._write(AceMessage.request.USERDATA(self._gender, self._age))
elif 'cansave' in self._tempevent: pass
elif 'showurl' in self._tempevent: pass
elif 'download_stopped' in self._tempevent: pass
# PAUSE
elif self._recvbuffer.startswith('PAUSE'): self._write(AceMessage.request.EVENT('pause'))
# RESUME
elif self._recvbuffer.startswith('RESUME'): self._write(AceMessage.request.EVENT('play'))
# STOP
elif self._recvbuffer.startswith('STOP'): pass
# SHUTDOWN
elif self._recvbuffer.startswith('SHUTDOWN'):
self._socket.close()
logger.debug('AceClient destroyed')
break
class GClient(object):
"""
A generic gevent-based network client, that implements common send and receive functionality.
Useful members:
group: A gevent.pool.Group() tied to the lifetime of the client. When stopping, all greenlets
in the group will be killed.
started: True if the client has been started
stopped: True if the client has been stopped
running: True if the client has been started but not stopped
"""
def __init__(self, logger=None):
self.group = Group()
self.started = False
self._send_queue = Queue()
self._stopping = False
self._stopped = AsyncResult()
if not hasattr(self, 'logger'): # let subclass overrride if they want
if not logger:
logger = logging.getLogger('gclient').getChild(type(self).__name__)
self.logger = logger
def start(self):
"""Start the client, performing some connection step and beginning processing."""
if self.started:
raise Exception("Already started")
self.started = True
self.logger.debug("{} starting".format(self))
self._start()
self._send_loop_worker = self.group.spawn(self._send_loop)
self._recv_loop_worker = self.group.spawn(self._recv_loop)
self.logger.info("{} started".format(self))
def _start(self):
"""Override this with code that creates and initializes a connection"""
def stop(self, ex=None):
"""Stop the client, optionally referencing some exception.
This will kill all greenlets in group and do any specific stop handling.
Anyone waiting on the client stopping will have the exception raised, if any.
"""
if self._stopping:
self.wait_for_stop()
return
if not self.started:
self.started = True
self._stopping = True
if ex:
self.logger.info("{} stopping with error".format(self), exc_info=True)
else:
self.logger.info("{} stopping".format(self))
# since the greenlet calling stop() might be in self.group, we make a new greenlet to do the work
@gevent.spawn
def stop_worker():
self.group.kill(block=True)
while not self._send_queue.empty():
msg, event = self._send_queue.get(block=False)
event.set()
self._stop(ex)
if ex:
self._stopped.set_exception(ex)
else:
self._stopped.set(None)
self.logger.debug("{} fully stopped".format(self))
stop_worker.get()
def _stop(self, ex=None):
"""Optionally override this with specific cleanup code for stopping the client,
such as closing the connection."""
pass
def wait_for_stop(self):
"""Block until the client has stopped, re-raising the exception it was stopped with, if any."""
self._stopped.get()
@property
def stopped(self):
return self._stopped.ready()
@property
def running(self):
return self.started and not self.stopped
def send(self, msg, block=False):
"""Enqueue some kind of message to be sent. If block=True, block until actually sent.
If block=False, returns a gevent.event.Event() that will be set when actually sent,
or the client is stopped.
Note that messages are sent in order, so using either of these shouldn't often be needed.
"""
if self._stopping:
raise Exception("Can't send to stopped client")
event = Event()
self._send_queue.put((msg, event))
if block:
event.wait()
else:
return event
def _send_loop(self):
try:
for msg, event in self._send_queue:
self._send(msg)
event.set()
except Exception as ex:
self.stop(ex)
def _send(self, msg):
"""Override this with specific code for sending a message. It may raise to indicate a failure
that will stop the client."""
def _recv_loop(self):
try:
self._receive()
except Exception as ex:
self.stop(ex)
else:
self.stop()
def _receive(self):
"""Override this with code that receives data. It may return to indicate a graceful close,
class Operation(BaseOperation):
def __init__(self, monitor, callable, *args):
super(Operation, self).__init__(monitor)
self.id = (callable, args)
self.callable = callable
self.args = args
self.node = getattr(callable, "__self__", None)
self.method = getattr(callable, "__name__", None)
self.result = AsyncResult()
self.primary_parent = self.monitor.get_current()
self.primary_parent.add_dependency(self)
self.greenlet = Yaylet(callable, *args)
self.greenlet.operation = self
self.greenlet.link(self._operation_finish)
def start(self):
self.greenlet.start()
def ready(self):
return self.result.ready()
def get(self):
return self.result.get()
def _operation_finish(self, source):
# WARNING: This method will be caused in it's own greenlet.
# Using self.monitor.execute from here will cause work to be owned by Root
# Cycle breaking
source.operation = None
# Setup the AsyncResult so *new* calls will return immediately
# But let's not notify the existing blocked greenlets until we
# have run the paradox detector
if source.successful():
self.result.value = source.value
self.result._exception = None
else:
self.result.value = None
self.result._exception = source.exception
# Purge any operations that were cached during a peek operation
checks = []
for p in self.peeks:
for c, op in p.walk_children():
if op.method.startswith("as_"):
checks.append(op)
op.purge_one()
if p.method.startswith("as_"):
checks.append(p)
p.purge_one()
getcurrent().operation = self
for op in checks:
try:
current_val = op.get()
new_val = self.monitor.wait(getattr(op.node, op.method))
except Exception as e:
self.result.set_exception(e)
return
if new_val != current_val:
self.result.set_exception(
errors.ParadoxError(
"Inconsistent configuration detected - changed from %r to %r" % (current_val, new_val),
anchor=op.node.anchor,
)
)
getcurrent().operation = None
return
getcurrent().operation = None
# Now notify all the other greenlets waiting for us that it is safe to continue
if source.successful():
self.result.set(source.value)
else:
self.result.set_exception(source.exception)
def __repr__(self):
return "%s<%s>.%s(%r)" % (self.node.__class__.__name__, id(self), self.method, self.args)
def new_netting_channel(
self,
partner: typing.Address,
settle_timeout: int,
) -> typing.ChannelID:
""" Creates a new channel in the TokenNetwork contract.
Args:
partner: The peer to open the channel with.
settle_timeout: The settle timout to use for this channel.
Returns:
The address of the new netting channel.
"""
if not is_binary_address(partner):
raise InvalidAddress('Expected binary address format for channel partner')
invalid_timeout = (
settle_timeout < self.settlement_timeout_min() or
settle_timeout > self.settlement_timeout_max()
)
if invalid_timeout:
raise InvalidSettleTimeout('settle_timeout must be in range [{}, {}], is {}'.format(
self.settlement_timeout_min(),
self.settlement_timeout_max(),
settle_timeout,
))
if self.node_address == partner:
raise SamePeerAddress('The other peer must not have the same address as the client.')
# Prevent concurrent attempts to open a channel with the same token and
# partner address.
if partner not in self.open_channel_transactions:
new_open_channel_transaction = AsyncResult()
self.open_channel_transactions[partner] = new_open_channel_transaction
try:
transaction_hash = self._new_netting_channel(partner, settle_timeout)
except Exception as e:
new_open_channel_transaction.set_exception(e)
raise
else:
new_open_channel_transaction.set(transaction_hash)
finally:
self.open_channel_transactions.pop(partner, None)
else:
# All other concurrent threads should block on the result of opening this channel
self.open_channel_transactions[partner].get()
channel_created = self.channel_exists(self.node_address, partner)
if channel_created is False:
log.error(
'creating new channel failed',
peer1=pex(self.node_address),
peer2=pex(partner),
)
raise RuntimeError('creating new channel failed')
channel_identifier = self.detail_channel(self.node_address, partner)['channel_identifier']
log.info(
'new_netting_channel called',
peer1=pex(self.node_address),
peer2=pex(partner),
channel_identifier=encode_hex(channel_identifier),
)
return channel_identifier
class AceClient:
def __init__(self, host, port, connect_timeout = 5, result_timeout = 5, debug = logging.ERROR):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Debug level
self._debug = debug
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Logger
logger = logging.getLogger('AceClient_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException("Socket creation error! Ace is not running? " + str(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def _write(self, message):
try:
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + str(e))
def aceInit(self, gender = AceConst.SEX_MALE, age = AceConst.AGE_18_24, product_key = None, pause_delay = 0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def START(self, datatype, value):
'''
Start video method
'''
# Logger
logger = logging.getLogger("AceClient_START")
self._result = AsyncResult()
self._urlresult = AsyncResult()
self._write(AceMessage.request.START(datatype.upper(), value))
try:
if not self._result.get(timeout = self._resulttimeout):
errmsg = "START error!"
logger.error(errmsg)
raise AceException(errmsg)
except gevent.Timeout:
errmsg = "START timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getUrl(self, timeout = 40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout = timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getPlayEvent(self, timeout = None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
self._resumeevent.wait(timeout = timeout)
return
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n", 1)
self._recvbuffer = self._recvbuffer.strip()
except:
# If something happened during read, abandon reader
# Should not ever happen
logger.error("Exception at socket read")
return
# Parsing everything
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'key=' in self._recvbuffer:
self._request_key = self._recvbuffer.split()[2].split('=')[1]
self._write(AceMessage.request.READY_key(self._request_key, self._product_key))
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
return
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to "+self._status)
if self._status == 'main:err':
logger.warning(self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
class AceClient(object):
def __init__(self, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = 0
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0
# Logger
logger = logging.getLogger('AceClieimport tracebacknt_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException("Socket creation error! Ace is not running? " +
repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger("AceClient_write")
logger.debug(message)
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_18_24,
product_key=None,
pause_delay=0,
seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException("Result not received")
except gevent.Timeout:
raise AceException("Timeout")
return result
def START(self, datatype, value):
'''
Start video method
'''
stream_type = 'output_format=http' if self._engine_version_code >= 3010500 and not AceConfig.vlcuse else ''
self._urlresult = AsyncResult()
self._write(
AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state is not None and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, url):
self._result = AsyncResult()
self._write(
AceMessage.request.LOADASYNC(datatype.upper(), 0, {'url': url}))
return self._getResult()
def GETCID(self, datatype, url):
contentinfo = self.LOADASYNC(datatype, url)
self._cidresult = AsyncResult()
self._write(
AceMessage.request.GETCID(contentinfo.get('checksum'),
contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def GETCONTENTINFO(self, datatype, url):
contentinfo = self.LOADASYNC(datatype, url)
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter):
logger = logging.getLogger("StreamReader")
self._streamReaderState = 1
logger.debug("Opening video stream: %s" % url)
try:
connection = self._streamReaderConnection = urllib2.urlopen(url)
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return
if connection.getcode() != 200:
logger.error("Failed to open video stream %s" % connection)
return
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
while True:
data = None
clients = counter.getClients(cid)
try:
data = connection.read(AceConfig.readchunksize)
except:
break
if data and clients:
with self._lock:
if len(self._streamReaderQueue
) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" %
str(c))
c.destroy()
elif not clients:
logger.debug(
"All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except urllib2.URLError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClients(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger("StreamReader")
c = self._streamReaderConnection
if c:
self._streamReaderConnection = None
c.close()
logger.debug("Video stream closed")
self._streamReaderQueue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
# logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'version_code=' in self._recvbuffer:
v = self._recvbuffer.find('version_code=')
self._engine_version_code = int(
self._recvbuffer[v + 13:v + 20])
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14]
try:
self._write(
AceMessage.request.READY_key(
self._request_key, self._product_key))
except urllib2.URLError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error(
"LOADASYNC returned error with message: %s" %
_contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
self._result.set(_contentinfo)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or self._started_again or not self._recvbuffer.endswith(
' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seeback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug("START received. Waiting for %s." %
AceMessage.response.LIVEPOS)
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(
self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(
AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
if self._seekback and not self._started_again:
self._write(AceMessage.request.SEEK(str(int(self._position_last) - \
self._seekback)))
logger.debug('Seeking back')
self._started_again = True
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(
';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(self._status + ' with message ' +
self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' +
self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' +
self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._status == 'main:idle':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
elif self._recvbuffer.startswith('##') or len(
self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
logger.debug("CID: %s" % self._recvbuffer)
def sendHeadersPT(self, client, code, headers):
client.handler.send_response(code)
for key, value in headers.items():
client.handler.send_header(key, value)
client.handler.end_headers()
def openStreamReaderPT(self, url, req_headers):
logger = logging.getLogger("openStreamReaderPT")
logger.debug("Opening video stream: %s" % url)
logger.debug("headers: %s" % req_headers)
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return None, None, None
request = urllib2.Request(url, headers=req_headers)
connection = self._streamReaderConnection = urllib2.urlopen(
request, timeout=120)
code = connection.getcode()
if code not in (200, 206):
logger.error("Failed to open video stream %s" % connection)
return None, None, None
FORWARD_HEADERS = [
'Content-Range',
'Connection',
'Keep-Alive',
'Content-Type',
'Accept-Ranges',
'X-Content-Duration',
'Content-Length',
]
SKIP_HEADERS = ['Server', 'Date']
response_headers = {}
for k in connection.info().headers:
if k.split(':')[0] not in (FORWARD_HEADERS + SKIP_HEADERS):
logger.debug('NEW HEADERS: %s' % k.split(':')[0])
for h in FORWARD_HEADERS:
if connection.info().getheader(h):
response_headers[h] = connection.info().getheader(h)
# self.connection.send_header(h, connection.info().getheader(h))
logger.debug('key=%s value=%s' %
(h, connection.info().getheader(h)))
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
return connection, code, response_headers
def startStreamReaderPT(self, url, cid, counter, req_headers=None):
logger = logging.getLogger("StreamReaderPT")
self._streamReaderState = 1
# current_req_headers = req_headers
try:
while True:
data = None
clients = counter.getClientsPT(cid)
if not req_headers == self.req_headers:
self.req_headers = req_headers
connection, code, resp_headers = self.openStreamReaderPT(
url, req_headers)
if not connection:
return
for c in clients:
try:
c.headers_sent
except:
self.sendHeadersPT(c, code, resp_headers)
c.headers_sent = True
# logger.debug("i")
try:
data = connection.read(AceConfig.readchunksize)
except:
break
# logger.debug("d: %s c:%s" % (data, clients))
if data and clients:
with self._lock:
if len(self._streamReaderQueue
) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" %
str(c))
c.destroy()
elif not clients:
logger.debug(
"All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except urllib2.URLError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClientsPT(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
class Bucket(AsyncBucket):
def __init__(self, *args, **kwargs):
"""
This class is a 'GEvent'-optimized subclass of libcouchbase
which utilizes the underlying IOPS structures and the gevent
event primitives to efficiently utilize couroutine switching.
"""
super(Bucket, self).__init__(IOPS(), *args, **kwargs)
def _do_ctor_connect(self):
if self.connected:
return
self._connect()
self._evconn = AsyncResult()
self._conncb = self._on_connected
self._evconn.get()
self._evconn = None
def _on_connected(self, err):
if err:
self._evconn.set_exception(err)
else:
self._evconn.set(None)
def _waitwrap(self, cbasync):
cur_thread = getcurrent()
errback = lambda r, x, y, z: cur_thread.throw(x, y, z)
cbasync.set_callbacks(cur_thread.switch, errback)
try:
return get_hub().switch()
finally:
# Deregister callbacks to prevent another request on the same
# greenlet to get the result from this context.
cbasync.set_callbacks(dummy_callback, dummy_callback)
def _meth_factory(meth, name):
def ret(self, *args, **kwargs):
return self._waitwrap(meth(self, *args, **kwargs))
return ret
def _http_request(self, **kwargs):
res = super(Bucket, self)._http_request(**kwargs)
w = Waiter()
res.callback = lambda x: w.switch(x)
res.errback = lambda x, c, o, b: w.throw(c, o, b)
return w.get()
def query(self, *args, **kwargs):
kwargs['itercls'] = GView
return super(Bucket, self).query(*args, **kwargs)
def n1ql_query(self, query, *args, **kwargs):
kwargs['itercls'] = GN1QLRequest
return super(Bucket, self).n1ql_query(query, *args, **kwargs)
def _get_close_future(self):
ev = Event()
def _dtor_cb(*args):
ev.set()
self._dtorcb = _dtor_cb
return ev
locals().update(AsyncBucket._gen_memd_wrappers(_meth_factory))
class StreamingExtract(object):
def __init__(self, id, hddsem, threadpool):
self.id = id
self.hddsem = hddsem
self.threadpool = threadpool
self.password = None
self.killed = False
self.parts = dict()
self.first = None
self.current = None
self.next = None
self.next_part_event = AsyncResult()
self.rar = None
extractors[id] = self
def feed_part(self, path, file):
path.finished = AsyncResult()
self.parts[path.path] = path, file
log.debug('fed new part {}: {}'.format(path, path))
if file.state != 'rarextract':
with transaction:
file.state = 'rarextract'
if self.first is None:
self.current = path, file
self.run(path, file)
else:
if path.path == self.next:
self.next_part_event.set(path)
path.finished.get()
def run(self, path, file):
try:
self.first = self.current
with transaction:
file.greenlet = gevent.getcurrent()
file.on_greenlet_started()
try:
result = self.bruteforce(path, file)
except rarfile.NeedFirstVolume:
self.next = os.path.join(path.dir, "{}.part{}.rar".format(path.basename, "1".zfill(len(path.part))))
return self.find_next()
if result and result is not True:
raise result
if self.password:
rarpw = "-p"+self.password
else:
rarpw = "-p-"
cmd = [rarfile.UNRAR_TOOL, "x", "-y", rarpw, "-idq", "-vp", path, file.get_extract_path() + os.sep]
file.log.info("starting extraction of {} with params {}".format(path[1:], cmd))
self.rar = rarfile.custom_popen(cmd)
self.wait_data()
if not path.finished.ready():
path.finished.set()
except BaseException as e:
traceback.print_exc()
self.kill(e)
raise
def bruteforce(self, path, file):
try:
rar = rarfile.RarFile(path, ignore_next_part_missing=True)
except rarfile.NeedFirstVolume:
raise
if not rar.needs_password():
self.password = None
return
if rar.needs_password() and rar.infolist():
# unencrypted headers. use file password or ask user.
pw = None
if len(file.package.extract_passwords) == 1:
pw = file.package.extract_passwords[0]
if not pw:
for pw in file.solve_password(message="Rarfile {} password cannot be cracked. Enter correct password: #".format(path.name), retries=1):
break
else:
return self.kill('extract password not entered')
self.password = pw
return
passwords = []
for i in itertools.chain(file.package.extract_passwords, core.config.bruteforce_passwords):
if not i in passwords:
passwords.append(i)
print "testing", passwords
if not self.threadpool.apply(bruteforce, (rar, passwords, self.hddsem, file.log)):
# ask user for password
for pw in file.solve_password(message="Enter the extract password for file: {} #".format(path.name), retries=5):
if self.threadpool.apply(bruteforce, (rar, [pw], self.hddsem, file.log)):
break
else:
return self.kill('extract password not entered')
self.password = rar._password
if self.password and self.password not in core.config.bruteforce_passwords:
with transaction:
core.config.bruteforce_passwords.append(self.password)
def wait_data(self):
bytes = ''
while True:
data = self.rar.stdout.read(1)
if not data:
break
bytes += data
for i in bytes.splitlines():
if i:
result = self.new_data(i)
if result is True:
bytes = ''
if result and result is not True:
raise result
self.close()
def finish_file(self, path, file):
if file is not None:
with core.transaction:
#if not 'rarextract' in file.completed_plugins:
# file.completed_plugins.append('rarextract')
#file.greenlet = None
#file.on_greenlet_finish()
#file.on_greenlet_stopped()
file.state = 'rarextract_complete'
file.init_progress(1)
file.set_progress(1)
#file.stop()
#path.finished.set()
event.fire('rarextract:part_complete', path, file)
def new_data(self, data):
"""called when new data or new line
"""
#print "got new data from unrar:", data
if "packed data CRC failed in volume" in data:
return self.kill('checksum error in rar archive')
if data.startswith("CRC failed in the encrypted file"): # corrupt file or download not complete
return self.kill('checksum error in rar archive. wrong password?')
m = re.search(r"Insert disk with (.*?) \[C\]ontinue\, \[Q\]uit", data)
if not m:
return
if self.current is not None:
self.finish_file(*self.current)
self.next = m.group(1)
print "setting self.next", self.next
return self.find_next()
def find_next(self):
print "finding next", self.next
next = self.next
if next not in self.parts:
# check if file is in core.files()
found = False
name = os.path.basename(next)
for f in core.files():
if f.name == name and f.get_complete_file() == next:
found = True
if not f.working and 'download' in f.completed_plugins:
current = fileplugin.FilePath(next), f
current[0].finished = AsyncResult()
self.parts[next] = current
log.debug('got next part from idle {}: {}'.format(next, self.current[0]))
break
if not found:
# file is not in system, check if it exists on hdd
if os.path.exists(next):
current = fileplugin.FilePath(next), self.first[1]
current[0].finished = AsyncResult()
self.parts[next] = current
log.debug('got next part from hdd {}: {}'.format(next, self.current[0]))
else:
# part does not exists. fail this extract
return self.kill('missing part {}'.format(next))
if next not in self.parts:
log.debug('waiting for part {}'.format(next))
event.fire('rarextract:waiting_for_part', next)
while next not in self.parts:
self.next_part_event.get()
self.next_part_event = AsyncResult()
log.debug('got next part from wait {}: {}'.format(next, self.current[0]))
self.current = self.parts[next]
return self.go_on()
def go_on(self):
if self.rar is None:
return self.run(*self.current)
if not os.path.exists(self.next):
return
self.rar.stdin.write("C\n")
self.rar.stdin.flush()
if self.current[1] is not None:
with core.transaction:
self.current[1].greenlet = gevent.getcurrent()
self.current[1].greenlet.link(self.current[0].finished)
self.current[1].on_greenlet_started()
self.current[1].log.info("extract go on: {}".format(self.current[1].name))
return True
def kill(self, exc=""):
#blacklist.add(self.first[0].basename) # no autoextract for failed archives
print "killing rarextract", self.first[0].basename
if isinstance(exc, basestring):
exc = ValueError(exc)
self.current = None
self.killed = True
if self.rar is not None:
self.rar.terminate()
self.rar = None
try:
del extractors[self.id]
except KeyError:
pass
self.next_part_event.set_exception(exc)
for path, file in self.parts.values():
if not path.finished.ready():
path.finished.set_exception(exc)
with transaction:
for path, file in self.parts.values():
if file is not None:
file.stop()
if file.state == 'rarextract_complete':
file.state = 'rarextract'
file.enabled = False
if 'rarextract' in file.completed_plugins:
file.completed_plugins.remove('rarextract')
self.first[1].fatal('rarextract: {}'.format(exc))
return exc
def close(self):
"""called when process is closed"""
try:
del extractors[self.id]
except KeyError:
pass
if not self.killed:
if self.current is not None:
self.finish_file(*self.current)
if core.config.delete_extracted_archives:
for path, file in self.parts.values():
os.remove(file.get_complete_file())
def register_secret_batch(self, secrets: List[Secret]):
secrets_to_register = list()
secrethashes_to_register = list()
secrethashes_not_sent = list()
secret_registry_transaction = AsyncResult()
for secret in secrets:
secrethash = sha3(secret)
secrethash_hex = encode_hex(secrethash)
is_register_needed = (not self.check_registered(secrethash) and
secret not in self.open_secret_transactions)
if is_register_needed:
secrets_to_register.append(secret)
secrethashes_to_register.append(secrethash_hex)
self.open_secret_transactions[
secret] = secret_registry_transaction
else:
secrethashes_not_sent.append(secrethash_hex)
log_details = {
'node': pex(self.node_address),
'contract': pex(self.address),
'secrethashes': secrethashes_to_register,
'secrethashes_not_sent': secrethashes_not_sent,
}
if not secrets_to_register:
log.debug('registerSecretBatch skipped', **log_details)
return
checking_block = self.client.get_checking_block()
error_prefix = 'Call to registerSecretBatch will fail'
gas_limit = self.proxy.estimate_gas(checking_block,
'registerSecretBatch', secrets)
if gas_limit:
error_prefix = 'Call to registerSecretBatch failed'
try:
gas_limit = safe_gas_limit(
gas_limit,
len(secrets) * GAS_REQUIRED_PER_SECRET_IN_BATCH,
)
transaction_hash = self.proxy.transact('registerSecretBatch',
gas_limit, secrets)
self.client.poll(transaction_hash)
receipt_or_none = check_transaction_threw(
self.client, transaction_hash)
except Exception as e:
secret_registry_transaction.set_exception(e)
msg = 'Unexpected exception at sending registerSecretBatch transaction'
else:
secret_registry_transaction.set(transaction_hash)
finally:
for secret in secrets_to_register:
self.open_secret_transactions.pop(secret, None)
transaction_executed = gas_limit is not None
if not transaction_executed or receipt_or_none:
if transaction_executed:
block = receipt_or_none['blockNumber']
else:
block = checking_block
self.proxy.jsonrpc_client.check_for_insufficient_eth(
transaction_name='registerSecretBatch',
transaction_executed=transaction_executed,
required_gas=len(secrets) * GAS_REQUIRED_PER_SECRET_IN_BATCH,
block_identifier=block,
)
error_msg = f'{error_prefix}. {msg}'
log.critical(error_msg, **log_details)
raise RaidenUnrecoverableError(error_msg)
log.info('registerSecretBatch successful', **log_details)
class AceClient(object):
def __init__(self, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = 0
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0;
# Logger
logger = logging.getLogger('AceClieimport tracebacknt_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException(
"Socket creation error! Ace is not running? " + repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger("AceClient_write")
logger.debug(message)
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0, seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException("Result not received")
except gevent.Timeout:
raise AceException("Timeout")
return result
def START(self, datatype, value):
'''
Start video method
'''
stream_type = 'output_format=http' if self._engine_version_code >= 3010500 and not AceConfig.vlcuse else ''
self._urlresult = AsyncResult()
self._write(AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state is not None and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, url):
self._result = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, {'url': url}))
return self._getResult()
def GETCID(self, datatype, url):
contentinfo = self.LOADASYNC(datatype, url)
self._cidresult = AsyncResult()
self._write(AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def GETCONTENTINFO(self, datatype, url):
contentinfo = self.LOADASYNC(datatype, url)
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter):
logger = logging.getLogger("StreamReader")
self._streamReaderState = 1
logger.debug("Opening video stream: %s" % url)
try:
connection = self._streamReaderConnection = urllib2.urlopen(url)
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return
if connection.getcode() != 200:
logger.error("Failed to open video stream %s" % connection)
return
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
while True:
data = None
clients = counter.getClients(cid)
try:
data = connection.read(AceConfig.readchunksize)
except:
break;
if data and clients:
with self._lock:
if len(self._streamReaderQueue) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" % str(c))
c.destroy()
elif not clients:
logger.debug("All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except urllib2.URLError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClients(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger("StreamReader")
c = self._streamReaderConnection
if c:
self._streamReaderConnection = None
c.close()
logger.debug("Video stream closed")
self._streamReaderQueue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
# logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'version_code=' in self._recvbuffer:
v = self._recvbuffer.find('version_code=')
self._engine_version_code = int(self._recvbuffer[v + 13:v + 20])
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except urllib2.URLError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
self._result.set(_contentinfo)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or self._started_again or not self._recvbuffer.endswith(' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seeback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS)
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
if self._seekback and not self._started_again:
self._write(AceMessage.request.SEEK(str(int(self._position_last) - \
self._seekback)))
logger.debug('Seeking back')
self._started_again = True
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._status == 'main:idle':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
elif self._recvbuffer.startswith('##') or len(self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
logger.debug("CID: %s" % self._recvbuffer)
def sendHeadersPT(self, client, code, headers):
client.handler.send_response(code)
for key, value in headers.items():
client.handler.send_header(key, value)
client.handler.end_headers()
def openStreamReaderPT(self, url, req_headers):
logger = logging.getLogger("openStreamReaderPT")
logger.debug("Opening video stream: %s" % url)
logger.debug("headers: %s" % req_headers)
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return None, None, None
request = urllib2.Request(url, headers=req_headers)
connection = self._streamReaderConnection = urllib2.urlopen(request, timeout=120)
code = connection.getcode()
if code not in (200, 206):
logger.error("Failed to open video stream %s" % connection)
return None, None, None
FORWARD_HEADERS = ['Content-Range',
'Connection',
'Keep-Alive',
'Content-Type',
'Accept-Ranges',
'X-Content-Duration',
'Content-Length',
]
SKIP_HEADERS = ['Server', 'Date']
response_headers = {}
for k in connection.info().headers:
if k.split(':')[0] not in (FORWARD_HEADERS + SKIP_HEADERS):
logger.debug('NEW HEADERS: %s' % k.split(':')[0])
for h in FORWARD_HEADERS:
if connection.info().getheader(h):
response_headers[h] = connection.info().getheader(h)
# self.connection.send_header(h, connection.info().getheader(h))
logger.debug('key=%s value=%s' % (h, connection.info().getheader(h)))
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
return connection, code, response_headers
def startStreamReaderPT(self, url, cid, counter, req_headers=None):
logger = logging.getLogger("StreamReaderPT")
self._streamReaderState = 1
# current_req_headers = req_headers
try:
while True:
data = None
clients = counter.getClientsPT(cid)
if not req_headers == self.req_headers:
self.req_headers = req_headers
connection, code, resp_headers = self.openStreamReaderPT(url, req_headers)
if not connection:
return
for c in clients:
try:
c.headers_sent
except:
self.sendHeadersPT(c, code, resp_headers)
c.headers_sent = True
# logger.debug("i")
try:
data = connection.read(AceConfig.readchunksize)
except:
break;
# logger.debug("d: %s c:%s" % (data, clients))
if data and clients:
with self._lock:
if len(self._streamReaderQueue) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" % str(c))
c.destroy()
elif not clients:
logger.debug("All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except urllib2.URLError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClientsPT(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
class AioFile(object):
_keep_awake_thread = None
_keep_awake_refs = 0
chunk_size = (16 << 10)
def __init__(self, path, tmp_dir=None):
self.path = path
self.tmp_dir = tmp_dir
self.event = AsyncResult()
@classmethod
def _start_keep_awake_thread(cls):
if not cls._keep_awake_thread:
cls._keep_awake_thread = gevent.spawn(cls._keep_awake)
cls._keep_awake_refs += 1
@classmethod
def _stop_keep_awake_thread(cls):
cls._keep_awake_refs -= 1
if cls._keep_awake_refs <= 0:
cls._keep_awake_thread.kill()
@classmethod
def _keep_awake(cls):
while True:
gevent.sleep(0.001)
def _write_callback(self, ret, errno):
if ret > 0:
self.event.set(ret)
else:
exc = IOError(errno, os.strerror(errno))
self.event.set_exception(exc)
def _write_piece(self, fd, data, data_len, offset):
remaining = data_len - offset
if remaining > self.chunk_size:
remaining = self.chunk_size
piece = data[offset:offset+remaining]
aio_write(fd, piece, offset, self._write_callback)
return self.event.get()
def dump(self, data):
data_view = memoryview(data)
data_len = len(data)
offset = 0
self._start_keep_awake_thread()
fd, filename = mkstemp(dir=self.tmp_dir)
try:
while True:
ret = self._write_piece(fd, data_view, data_len, offset)
offset += ret
if offset >= data_len:
break
os.rename(filename, self.path)
finally:
os.close(fd)
self._stop_keep_awake_thread()
def pickle_dump(self, obj):
return self.dump(cPickle.dumps(obj, cPickle.HIGHEST_PROTOCOL))
def _read_callback(self, buf, ret, errno):
if ret > 0:
self.event.set(buf)
elif ret == 0:
exc = EOFError()
self.event.set_exception(exc)
else:
exc = IOError(errno, os.strerror(errno))
self.event.set_exception(exc)
def _read_piece(self, fd, offset):
aio_read(fd, offset, self.chunk_size, self._read_callback)
return self.event.get()
def load(self):
data = bytearray()
offset = 0
self._start_keep_awake_thread()
fd = os.open(self.path, os.O_RDONLY)
try:
while True:
buf = self._read_piece(fd, offset)
offset += len(buf)
data.extend(buf)
except EOFError:
return str(data)
finally:
os.close(fd)
self._stop_keep_awake_thread()
def pickle_load(self):
return cPickle.loads(self.load())
class TaskExecutor(object):
def __init__(self, balancer, index):
self.balancer = balancer
self.index = index
self.task = None
self.proc = None
self.pid = None
self.conn = None
self.state = WorkerState.STARTING
self.key = str(uuid.uuid4())
self.result = AsyncResult()
self.exiting = False
self.killed = False
self.thread = gevent.spawn(self.executor)
self.cv = Condition()
self.status_lock = RLock()
def checkin(self, conn):
with self.cv:
self.balancer.logger.debug("Check-in of worker #{0} (key {1})".format(self.index, self.key))
self.conn = conn
self.state = WorkerState.IDLE
self.cv.notify_all()
def put_progress(self, progress):
st = TaskStatus(None)
st.__setstate__(progress)
self.task.set_state(progress=st)
def put_status(self, status):
with self.cv:
# Try to collect rusage at this point, when process is still alive
try:
kinfo = self.balancer.dispatcher.threaded(bsd.kinfo_getproc, self.pid)
self.task.rusage = kinfo.rusage
except LookupError:
pass
if status["status"] == "ROLLBACK":
self.task.set_state(TaskState.ROLLBACK)
if status["status"] == "FINISHED":
self.result.set(status["result"])
if status["status"] == "FAILED":
error = status["error"]
if error["type"] in ERROR_TYPES:
cls = ERROR_TYPES[error["type"]]
exc = cls(
code=error["code"],
message=error["message"],
stacktrace=error["stacktrace"],
extra=error.get("extra"),
)
else:
exc = OtherException(
code=error["code"],
message=error["message"],
stacktrace=error["stacktrace"],
type=error["type"],
extra=error.get("extra"),
)
self.result.set_exception(exc)
def put_warning(self, warning):
self.task.add_warning(warning)
def run(self, task):
def match_file(module, f):
name, ext = os.path.splitext(f)
return module == name and ext in [".py", ".pyc", ".so"]
with self.cv:
self.cv.wait_for(lambda: self.state == WorkerState.ASSIGNED)
self.result = AsyncResult()
self.task = task
self.task.set_state(TaskState.EXECUTING)
self.state = WorkerState.EXECUTING
self.cv.notify_all()
self.balancer.logger.debug("Actually starting task {0}".format(task.id))
filename = None
module_name = inspect.getmodule(task.clazz).__name__
for dir in self.balancer.dispatcher.plugin_dirs:
found = False
try:
for root, _, files in os.walk(dir):
file = first_or_default(lambda f: match_file(module_name, f), files)
if file:
filename = os.path.join(root, file)
found = True
break
if found:
break
except OSError:
continue
try:
self.conn.call_sync(
"taskproxy.run",
{
"id": task.id,
"user": task.user,
"class": task.clazz.__name__,
"filename": filename,
"args": task.args,
"debugger": task.debugger,
"environment": task.environment,
"hooks": task.hooks,
},
)
except RpcException as e:
self.balancer.logger.warning(
"Cannot start task {0} on executor #{1}: {2}".format(task.id, self.index, str(e))
)
self.balancer.logger.warning(
"Killing unresponsive task executor #{0} (pid {1})".format(self.index, self.proc.pid)
)
self.terminate()
try:
self.result.get()
except BaseException as e:
if isinstance(e, OtherException):
self.balancer.dispatcher.report_error("Task {0} raised invalid exception".format(self.task.name), e)
if isinstance(e, TaskAbortException):
self.task.set_state(TaskState.ABORTED, TaskStatus(0, "aborted"))
else:
self.task.error = serialize_error(e)
self.task.set_state(
TaskState.FAILED, TaskStatus(0, str(e), extra={"stacktrace": traceback.format_exc()})
)
with self.cv:
self.task.ended.set()
if self.state == WorkerState.EXECUTING:
self.state = WorkerState.IDLE
self.cv.notify_all()
self.balancer.task_exited(self.task)
return
with self.cv:
self.task.result = self.result.value
self.task.set_state(TaskState.FINISHED, TaskStatus(100, ""))
self.task.ended.set()
if self.state == WorkerState.EXECUTING:
self.state = WorkerState.IDLE
self.cv.notify_all()
self.balancer.task_exited(self.task)
def abort(self):
self.balancer.logger.info("Trying to abort task #{0}".format(self.task.id))
# Try to abort via RPC. If this fails, kill process
try:
# If task supports abort protocol we don't need to worry about subtasks - it's task
# responsibility to kill them
self.conn.call_sync("taskproxy.abort")
except RpcException as err:
self.balancer.logger.warning("Failed to abort task #{0} gracefully: {1}".format(self.task.id, str(err)))
self.balancer.logger.warning("Killing process {0}".format(self.pid))
self.killed = True
self.terminate()
# Now kill all the subtasks
for subtask in filter(lambda t: t.parent is self.task, self.balancer.task_list):
self.balancer.logger.warning(
"Aborting subtask {0} because parent task {1} died".format(subtask.id, self.task.id)
)
self.balancer.abort(subtask.id)
def terminate(self):
try:
self.proc.terminate()
except OSError:
self.balancer.logger.warning("Executor process with PID {0} already dead".format(self.proc.pid))
def executor(self):
while not self.exiting:
try:
self.proc = Popen(
[TASKWORKER_PATH, self.key],
close_fds=True,
preexec_fn=os.setpgrp,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
self.pid = self.proc.pid
self.balancer.logger.debug("Started executor #{0} as PID {1}".format(self.index, self.pid))
except OSError:
self.result.set_exception(TaskException(errno.EFAULT, "Cannot spawn task executor"))
self.balancer.logger.error("Cannot spawn task executor #{0}".format(self.index))
return
for line in self.proc.stdout:
line = line.decode("utf8")
self.balancer.logger.debug("Executor #{0}: {1}".format(self.index, line.strip()))
if self.task:
self.task.output += line
self.proc.wait()
with self.cv:
self.state = WorkerState.STARTING
self.cv.notify_all()
if self.proc.returncode == -signal.SIGTERM:
self.balancer.logger.info(
"Executor process with PID {0} was terminated gracefully".format(self.proc.pid)
)
else:
self.balancer.logger.error(
"Executor process with PID {0} died abruptly with exit code {1}".format(
self.proc.pid, self.proc.returncode
)
)
if self.killed:
self.result.set_exception(TaskException(errno.EFAULT, "Task killed"))
else:
self.result.set_exception(TaskException(errno.EFAULT, "Task executor died"))
gevent.sleep(1)
def die(self):
self.exiting = True
if self.proc:
self.terminate()
def _register_secret_batch(
self,
secrets_to_register: List[Secret],
transaction_result: AsyncResult,
log_details: Dict[str, Any],
) -> None:
estimated_transaction = self.client.estimate_gas(
self.proxy, "registerSecretBatch", log_details, secrets_to_register
)
msg = None
transaction_mined = None
if estimated_transaction is not None:
gas_limit = safe_gas_limit(
GAS_REQUIRED_REGISTER_SECRET_BATCH_BASE
+ len(secrets_to_register) * GAS_REQUIRED_PER_SECRET_IN_BATCH,
)
assert estimated_transaction.estimated_gas <= gas_limit, (
f"Our safe gas calculation must be larger than the gas cost estimated by the "
f"ethereum node, but {estimated_transaction.estimated_gas} > {gas_limit}."
)
estimated_transaction.estimated_gas = gas_limit
try:
transaction_sent = self.client.transact(estimated_transaction)
transaction_mined = self.client.poll_transaction(transaction_sent)
except Exception as e: # pylint: disable=broad-except
msg = f"Unexpected exception {e} at sending registerSecretBatch transaction."
# Clear `open_secret_transactions` regardless of the transaction being
# successfully executed or not.
with self._open_secret_transactions_lock:
for secret in secrets_to_register:
self.open_secret_transactions.pop(secret)
# As of version `0.4.0` of the contract has *no* asserts or requires.
# Therefore the only reason for the transaction to fail is if there is
# a bug.
unrecoverable_error = transaction_mined is None or not was_transaction_successfully_mined(
transaction_mined
)
exception: Union[RaidenRecoverableError, RaidenUnrecoverableError]
if unrecoverable_error:
# If the transaction was sent it must not fail. If this happened
# some of our assumptions is broken therefore the error is
# unrecoverable
if transaction_mined is not None:
receipt = transaction_mined.receipt
if receipt["gasUsed"] == transaction_mined.startgas:
# The transaction failed and all gas was used. This can
# happen because of:
#
# - A compiler bug if an invalid opcode was executed.
# - A configuration bug if an assert was executed,
# because version 0.4.0 of the secret registry does not have an
# assert.
# - An ethereum client bug if the gas_limit was
# underestimated.
#
# Safety cannot be guaranteed under any of these cases,
# this error is unrecoverable.
error = (
"Secret registration failed because of a bug in either "
"the solidity compiler, the running ethereum client, or "
"a configuration error in Raiden."
)
else:
# The transaction failed and *not* all gas was used. This
# can happen because of:
#
# - A compiler bug if a revert was introduced.
# - A configuration bug, because for 0.4.0 the secret
# registry does not have a revert.
error = (
"Secret registration failed because of a configuration "
"bug or compiler bug. Please double check the secret "
"smart contract is at version 0.4.0, if it is then a "
"compiler bug was hit."
)
exception = RaidenUnrecoverableError(error)
transaction_result.set_exception(exception)
raise exception
# If gas_limit is set and there is no receipt then an exception was
# raised while sending the transaction. This should only happen if
# the account is being used concurrently, which is not supported.
# This can happen because:
#
# - The nonce of the transaction was already used
# - The nonce was reused *and* the account didn't have enough ether
# to pay for the gas
#
# Safety cannot be guaranteed under any of these cases, this error
# is unrecoverable. *Note*: This assumes the ethereum client
# takes into account the current transactions in the pool.
if estimated_transaction is not None:
assert msg, "Unexpected control flow, an exception should have been raised."
error = (
f"Sending the transaction for registerSecretBatch "
f"failed with: `{msg}`. This happens if the same ethereum "
f"account is being used by more than one program which is not "
f"supported."
)
exception = RaidenUnrecoverableError(error)
transaction_result.set_exception(exception)
raise exception
# gas_limit can fail because:
#
# - The Ethereum client detected the transaction could not
# successfully execute, this happens if an assert/revert is hit.
# - The account is lacking funds to pay for the gas.
#
# Either of these is a bug. The contract does not use
# assert/revert, and the account should always be funded
self.client.check_for_insufficient_eth(
transaction_name="registerSecretBatch",
transaction_executed=True,
required_gas=GAS_REQUIRED_PER_SECRET_IN_BATCH * len(secrets_to_register),
block_identifier=self.client.get_checking_block(),
)
error = "Call to registerSecretBatch couldn't be done"
exception = RaidenRecoverableError(error)
transaction_result.set_exception(exception)
raise exception
# The local **MUST** transaction_result be set before waiting for the
# other results, otherwise we have a dead-lock
assert transaction_mined is not None, MYPY_ANNOTATION
transaction_result.set(transaction_mined.transaction_hash)
class Bucket(AsyncBucket):
def __init__(self, *args, **kwargs):
"""
This class is a 'GEvent'-optimized subclass of libcouchbase
which utilizes the underlying IOPS structures and the gevent
event primitives to efficiently utilize couroutine switching.
"""
super(Bucket, self).__init__(IOPS(), *args, **kwargs)
def _do_ctor_connect(self):
if self.connected:
return
self._connect()
self._evconn = AsyncResult()
self._conncb = self._on_connected
self._evconn.get()
self._evconn = None
def _on_connected(self, err):
if err:
self._evconn.set_exception(err)
else:
self._evconn.set(None)
def _waitwrap(self, cbasync):
cur_thread = getcurrent()
errback = lambda r, x, y, z: cur_thread.throw(x, y, z)
cbasync.set_callbacks(cur_thread.switch, errback)
try:
return get_hub().switch()
finally:
# Deregister callbacks to prevent another request on the same
# greenlet to get the result from this context.
cbasync.set_callbacks(dummy_callback, dummy_callback)
def _meth_factory(meth, name):
def ret(self, *args, **kwargs):
return self._waitwrap(meth(self, *args, **kwargs))
return ret
def _http_request(self, **kwargs):
res = super(Bucket, self)._http_request(**kwargs)
w = Waiter()
res.callback = lambda x: w.switch(x)
res.errback = lambda x, c, o, b: w.throw(c, o, b)
return w.get()
def query(self, *args, **kwargs):
kwargs['itercls'] = GView
return super(Bucket, self).query(*args, **kwargs)
def n1ql_query(self, query, *args, **kwargs):
kwargs['itercls'] = GN1QLRequest
return super(Bucket, self).n1ql_query(query, *args, **kwargs)
def _get_close_future(self):
ev = Event()
def _dtor_cb(*args):
ev.set()
self._dtorcb = _dtor_cb
return ev
locals().update(AsyncBucket._gen_memd_wrappers(_meth_factory))
def new_netting_channel(
self,
partner: typing.Address,
settle_timeout: int,
) -> typing.ChannelID:
""" Creates a new channel in the TokenNetwork contract.
Args:
partner: The peer to open the channel with.
settle_timeout: The settle timout to use for this channel.
Returns:
The ChannelID of the new netting channel.
"""
if not is_binary_address(partner):
raise InvalidAddress(
'Expected binary address format for channel partner')
invalid_timeout = (settle_timeout < self.settlement_timeout_min()
or settle_timeout > self.settlement_timeout_max())
if invalid_timeout:
raise InvalidSettleTimeout(
'settle_timeout must be in range [{}, {}], is {}'.format(
self.settlement_timeout_min(),
self.settlement_timeout_max(),
settle_timeout,
))
if self.node_address == partner:
raise SamePeerAddress(
'The other peer must not have the same address as the client.')
log_details = {
'peer1': pex(self.node_address),
'peer2': pex(partner),
}
log.debug('new_netting_channel called', **log_details)
# Prevent concurrent attempts to open a channel with the same token and
# partner address.
if partner not in self.open_channel_transactions:
new_open_channel_transaction = AsyncResult()
self.open_channel_transactions[
partner] = new_open_channel_transaction
try:
transaction_hash = self._new_netting_channel(
partner, settle_timeout)
except Exception as e:
log.critical('new_netting_channel failed', **log_details)
new_open_channel_transaction.set_exception(e)
raise
else:
new_open_channel_transaction.set(transaction_hash)
finally:
self.open_channel_transactions.pop(partner, None)
else:
# All other concurrent threads should block on the result of opening this channel
self.open_channel_transactions[partner].get()
channel_created = self.channel_exists_and_not_settled(
self.node_address, partner)
if channel_created is False:
log.critical('new_netting_channel failed', **log_details)
raise RaidenUnrecoverableError('creating new channel failed')
channel_identifier = self.detail_channel(self.node_address,
partner).channel_identifier
log_details['channel_identifier'] = channel_identifier
log.info('new_netting_channel successful', **log_details)
return channel_identifier
class _Socket(_original_Socket):
"""Green version of :class:`zmq.core.socket.Socket`
The following methods are overridden:
* send
* recv
To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or receiving
is deferred to the hub if a ``zmq.EAGAIN`` (retry) error is raised.
The `__state_changed` method is triggered when the zmq.FD for the socket is
marked as readable and triggers the necessary read and write events (which
are waited for in the recv and send methods).
Some double underscore prefixes are used to minimize pollution of
:class:`zmq.core.socket.Socket`'s namespace.
"""
__in_send_multipart = False
__in_recv_multipart = False
__writable = None
__readable = None
_state_event = None
_gevent_bug_timeout = 11.6 # timeout for not trusting gevent
_debug_gevent = False # turn on if you think gevent is missing events
_poller_class = _Poller
def __init__(self, context, socket_type):
_original_Socket.__init__(self, context, socket_type)
self.__in_send_multipart = False
self.__in_recv_multipart = False
self.__setup_events()
def __del__(self):
self.close()
def close(self, linger=None):
super(_Socket, self).close(linger)
self.__cleanup_events()
def __cleanup_events(self):
# close the _state_event event, keeps the number of active file descriptors down
if getattr(self, '_state_event', None):
_stop(self._state_event)
self._state_event = None
# if the socket has entered a close state resume any waiting greenlets
self.__writable.set()
self.__readable.set()
def __setup_events(self):
self.__readable = AsyncResult()
self.__writable = AsyncResult()
self.__readable.set()
self.__writable.set()
try:
self._state_event = get_hub().loop.io(self.getsockopt(zmq.FD), 1) # read state watcher
self._state_event.start(self.__state_changed)
except AttributeError:
# for gevent<1.0 compatibility
from gevent.core import read_event
self._state_event = read_event(self.getsockopt(zmq.FD), self.__state_changed, persist=True)
def __state_changed(self, event=None, _evtype=None):
if self.closed:
self.__cleanup_events()
return
try:
# avoid triggering __state_changed from inside __state_changed
events = super(_Socket, self).getsockopt(zmq.EVENTS)
except zmq.ZMQError as exc:
self.__writable.set_exception(exc)
self.__readable.set_exception(exc)
else:
if events & zmq.POLLOUT:
self.__writable.set()
if events & zmq.POLLIN:
self.__readable.set()
def _wait_write(self):
assert self.__writable.ready(), "Only one greenlet can be waiting on this event"
self.__writable = AsyncResult()
# timeout is because libzmq cannot be trusted to properly signal a new send event:
# this is effectively a maximum poll interval of 1s
tic = time.time()
dt = self._gevent_bug_timeout
if dt:
timeout = gevent.Timeout(seconds=dt)
else:
timeout = None
try:
if timeout:
timeout.start()
self.__writable.get(block=True)
except gevent.Timeout as t:
if t is not timeout:
raise
toc = time.time()
# gevent bug: get can raise timeout even on clean return
# don't display zmq bug warning for gevent bug (this is getting ridiculous)
if self._debug_gevent and timeout and toc-tic > dt and \
self.getsockopt(zmq.EVENTS) & zmq.POLLOUT:
print("BUG: gevent may have missed a libzmq send event on %i!" % self.FD, file=sys.stderr)
finally:
if timeout:
timeout.cancel()
self.__writable.set()
def _wait_read(self):
assert self.__readable.ready(), "Only one greenlet can be waiting on this event"
self.__readable = AsyncResult()
# timeout is because libzmq cannot always be trusted to play nice with libevent.
# I can only confirm that this actually happens for send, but lets be symmetrical
# with our dirty hacks.
# this is effectively a maximum poll interval of 1s
tic = time.time()
dt = self._gevent_bug_timeout
if dt:
timeout = gevent.Timeout(seconds=dt)
else:
timeout = None
try:
if timeout:
timeout.start()
self.__readable.get(block=True)
except gevent.Timeout as t:
if t is not timeout:
raise
toc = time.time()
# gevent bug: get can raise timeout even on clean return
# don't display zmq bug warning for gevent bug (this is getting ridiculous)
if self._debug_gevent and timeout and toc-tic > dt and \
self.getsockopt(zmq.EVENTS) & zmq.POLLIN:
print("BUG: gevent may have missed a libzmq recv event on %i!" % self.FD, file=sys.stderr)
finally:
if timeout:
timeout.cancel()
self.__readable.set()
def send(self, data, flags=0, copy=True, track=False):
"""send, which will only block current greenlet
state_changed always fires exactly once (success or fail) at the
end of this method.
"""
# if we're given the NOBLOCK flag act as normal and let the EAGAIN get raised
if flags & zmq.NOBLOCK:
try:
msg = super(_Socket, self).send(data, flags, copy, track)
finally:
if not self.__in_send_multipart:
self.__state_changed()
return msg
# ensure the zmq.NOBLOCK flag is part of flags
flags |= zmq.NOBLOCK
while True: # Attempt to complete this operation indefinitely, blocking the current greenlet
try:
# attempt the actual call
msg = super(_Socket, self).send(data, flags, copy, track)
except zmq.ZMQError as e:
# if the raised ZMQError is not EAGAIN, reraise
if e.errno != zmq.EAGAIN:
if not self.__in_send_multipart:
self.__state_changed()
raise
else:
if not self.__in_send_multipart:
self.__state_changed()
return msg
# defer to the event loop until we're notified the socket is writable
self._wait_write()
def recv(self, flags=0, copy=True, track=False):
"""recv, which will only block current greenlet
state_changed always fires exactly once (success or fail) at the
end of this method.
"""
if flags & zmq.NOBLOCK:
try:
msg = super(_Socket, self).recv(flags, copy, track)
finally:
if not self.__in_recv_multipart:
self.__state_changed()
return msg
flags |= zmq.NOBLOCK
while True:
try:
msg = super(_Socket, self).recv(flags, copy, track)
except zmq.ZMQError as e:
if e.errno != zmq.EAGAIN:
if not self.__in_recv_multipart:
self.__state_changed()
raise
else:
if not self.__in_recv_multipart:
self.__state_changed()
return msg
self._wait_read()
def send_multipart(self, *args, **kwargs):
"""wrap send_multipart to prevent state_changed on each partial send"""
self.__in_send_multipart = True
try:
msg = super(_Socket, self).send_multipart(*args, **kwargs)
finally:
self.__in_send_multipart = False
self.__state_changed()
return msg
def recv_multipart(self, *args, **kwargs):
"""wrap recv_multipart to prevent state_changed on each partial recv"""
self.__in_recv_multipart = True
try:
msg = super(_Socket, self).recv_multipart(*args, **kwargs)
finally:
self.__in_recv_multipart = False
self.__state_changed()
return msg
def get(self, opt):
"""trigger state_changed on getsockopt(EVENTS)"""
optval = super(_Socket, self).get(opt)
if opt == zmq.EVENTS:
self.__state_changed()
return optval
class _Socket(_original_Socket):
"""Green version of :class:`zmq.Socket`
The following methods are overridden:
* send
* recv
To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or receiving
is deferred to the hub if a ``zmq.EAGAIN`` (retry) error is raised.
The `__state_changed` method is triggered when the zmq.FD for the socket is
marked as readable and triggers the necessary read and write events (which
are waited for in the recv and send methods).
Some double underscore prefixes are used to minimize pollution of
:class:`zmq.Socket`'s namespace.
"""
__in_send_multipart = False
__in_recv_multipart = False
__writable = None
__readable = None
_state_event = None
_gevent_bug_timeout = 11.6 # timeout for not trusting gevent
_debug_gevent = False # turn on if you think gevent is missing events
_poller_class = _Poller
_repr_cls = "zmq.green.Socket"
def __init__(self, *a, **kw):
super(_Socket, self).__init__(*a, **kw)
self.__in_send_multipart = False
self.__in_recv_multipart = False
self.__setup_events()
def __del__(self):
self.close()
def close(self, linger=None):
super(_Socket, self).close(linger)
self.__cleanup_events()
def __cleanup_events(self):
# close the _state_event event, keeps the number of active file descriptors down
if getattr(self, '_state_event', None):
_stop(self._state_event)
self._state_event = None
# if the socket has entered a close state resume any waiting greenlets
self.__writable.set()
self.__readable.set()
def __setup_events(self):
self.__readable = AsyncResult()
self.__writable = AsyncResult()
self.__readable.set()
self.__writable.set()
try:
self._state_event = get_hub().loop.io(self.getsockopt(zmq.FD),
1) # read state watcher
self._state_event.start(self.__state_changed)
except AttributeError:
# for gevent<1.0 compatibility
from gevent.core import read_event
self._state_event = read_event(self.getsockopt(zmq.FD),
self.__state_changed,
persist=True)
def __state_changed(self, event=None, _evtype=None):
if self.closed:
self.__cleanup_events()
return
try:
# avoid triggering __state_changed from inside __state_changed
events = super(_Socket, self).getsockopt(zmq.EVENTS)
except zmq.ZMQError as exc:
self.__writable.set_exception(exc)
self.__readable.set_exception(exc)
else:
if events & zmq.POLLOUT:
self.__writable.set()
if events & zmq.POLLIN:
self.__readable.set()
def _wait_write(self):
assert self.__writable.ready(
), "Only one greenlet can be waiting on this event"
self.__writable = AsyncResult()
# timeout is because libzmq cannot be trusted to properly signal a new send event:
# this is effectively a maximum poll interval of 1s
tic = time.time()
dt = self._gevent_bug_timeout
if dt:
timeout = gevent.Timeout(seconds=dt)
else:
timeout = None
try:
if timeout:
timeout.start()
self.__writable.get(block=True)
except gevent.Timeout as t:
if t is not timeout:
raise
toc = time.time()
# gevent bug: get can raise timeout even on clean return
# don't display zmq bug warning for gevent bug (this is getting ridiculous)
if (self._debug_gevent and timeout and toc - tic > dt
and self.getsockopt(zmq.EVENTS) & zmq.POLLOUT):
print(
"BUG: gevent may have missed a libzmq send event on %i!" %
self.FD,
file=sys.stderr,
)
finally:
if timeout:
timeout.close()
self.__writable.set()
def _wait_read(self):
assert self.__readable.ready(
), "Only one greenlet can be waiting on this event"
self.__readable = AsyncResult()
# timeout is because libzmq cannot always be trusted to play nice with libevent.
# I can only confirm that this actually happens for send, but lets be symmetrical
# with our dirty hacks.
# this is effectively a maximum poll interval of 1s
tic = time.time()
dt = self._gevent_bug_timeout
if dt:
timeout = gevent.Timeout(seconds=dt)
else:
timeout = None
try:
if timeout:
timeout.start()
self.__readable.get(block=True)
except gevent.Timeout as t:
if t is not timeout:
raise
toc = time.time()
# gevent bug: get can raise timeout even on clean return
# don't display zmq bug warning for gevent bug (this is getting ridiculous)
if (self._debug_gevent and timeout and toc - tic > dt
and self.getsockopt(zmq.EVENTS) & zmq.POLLIN):
print(
"BUG: gevent may have missed a libzmq recv event on %i!" %
self.FD,
file=sys.stderr,
)
finally:
if timeout:
timeout.close()
self.__readable.set()
def send(self, data, flags=0, copy=True, track=False, **kwargs):
"""send, which will only block current greenlet
state_changed always fires exactly once (success or fail) at the
end of this method.
"""
# if we're given the NOBLOCK flag act as normal and let the EAGAIN get raised
if flags & zmq.NOBLOCK:
try:
msg = super(_Socket, self).send(data, flags, copy, track,
**kwargs)
finally:
if not self.__in_send_multipart:
self.__state_changed()
return msg
# ensure the zmq.NOBLOCK flag is part of flags
flags |= zmq.NOBLOCK
while (
True
): # Attempt to complete this operation indefinitely, blocking the current greenlet
try:
# attempt the actual call
msg = super(_Socket, self).send(data, flags, copy, track)
except zmq.ZMQError as e:
# if the raised ZMQError is not EAGAIN, reraise
if e.errno != zmq.EAGAIN:
if not self.__in_send_multipart:
self.__state_changed()
raise
else:
if not self.__in_send_multipart:
self.__state_changed()
return msg
# defer to the event loop until we're notified the socket is writable
self._wait_write()
def recv(self, flags=0, copy=True, track=False):
"""recv, which will only block current greenlet
state_changed always fires exactly once (success or fail) at the
end of this method.
"""
if flags & zmq.NOBLOCK:
try:
msg = super(_Socket, self).recv(flags, copy, track)
finally:
if not self.__in_recv_multipart:
self.__state_changed()
return msg
flags |= zmq.NOBLOCK
while True:
try:
msg = super(_Socket, self).recv(flags, copy, track)
except zmq.ZMQError as e:
if e.errno != zmq.EAGAIN:
if not self.__in_recv_multipart:
self.__state_changed()
raise
else:
if not self.__in_recv_multipart:
self.__state_changed()
return msg
self._wait_read()
def send_multipart(self, *args, **kwargs):
"""wrap send_multipart to prevent state_changed on each partial send"""
self.__in_send_multipart = True
try:
msg = super(_Socket, self).send_multipart(*args, **kwargs)
finally:
self.__in_send_multipart = False
self.__state_changed()
return msg
def recv_multipart(self, *args, **kwargs):
"""wrap recv_multipart to prevent state_changed on each partial recv"""
self.__in_recv_multipart = True
try:
msg = super(_Socket, self).recv_multipart(*args, **kwargs)
finally:
self.__in_recv_multipart = False
self.__state_changed()
return msg
def get(self, opt):
"""trigger state_changed on getsockopt(EVENTS)"""
if opt in TIMEOS:
warnings.warn("TIMEO socket options have no effect in zmq.green",
UserWarning)
optval = super(_Socket, self).get(opt)
if opt == zmq.EVENTS:
self.__state_changed()
return optval
def set(self, opt, val):
"""set socket option"""
if opt in TIMEOS:
warnings.warn("TIMEO socket options have no effect in zmq.green",
UserWarning)
return super(_Socket, self).set(opt, val)
