class _FormationCache(object):
"""A cache of instance data for a formation."""
def __init__(self, client, form_name, factory, interval):
self.client = client
self.form_name = form_name
self.factory = factory
self.interval = interval
self._gthread = None
self._cache = {}
self._stopped = Event()
self._running = Event()
def start(self):
self._gthread = gevent.spawn(self._loop)
self._running.wait(timeout=0.1)
return self
def stop(self, timeout=None):
self._stopped.set()
self._gthread.join(timeout)
def _update(self):
self._cache = self.client.query_formation(self.form_name,
self.factory)
def _loop(self):
while not self._stopped.isSet():
self._update()
self._running.set()
self._stopped.wait(self.interval)
def query(self):
"""Return all instances and their names."""
return dict(self._cache)
class SyncBaseDeamon(object):
def __init__(self, interval=5):
self.ip_addr = socket.gethostbyname(socket.gethostname())
self.pid = os.getpid()
self.channel = Queue(1)
self.sync_interval = int(interval)
self.exit_flag = Event()
self.lock_path = ""
self.logger = logging.getLogger()
def exit(self):
if not self.exit_flag.isSet():
self.exit_flag.set()
self.channel.put('exit')
else:
self.logger.warn("Waiting exit signal been resolved")
def terminate(self, code):
zk.close()
sys.exit(code)
def run(self):
greenlets = []
try:
zk.create(
self.lock_path,
"%s:%s" % (self.ip_addr, self.pid),
ephemeral=True,
makepath=True
)
except NodeExistsError:
ret, _ = zk.get(self.lock_path)
ip, pid = ret.split(':')
self.logger.error("Servers syncing process is running on %s, pid: %s" % (ip, pid))
self.terminate(EXIT_CODES['leave_alone'])
gevent.signal(signal.SIGTERM, self.exit)
gevent.signal(signal.SIGINT, self.exit)
gevent.signal(signal.SIGHUP, self.exit)
greenlets.append(gevent.spawn(self.sync))
gevent.spawn(self.ticker)
gevent.joinall(greenlets)
self.logger.warn("syncing deamon exited")
self.terminate(EXIT_CODES['normal_exit'])
def ticker(self):
while True:
try:
if self.channel.qsize() == 0:
self.channel.put("sync", block=False)
except Full:
self.logger.error("something wrong occur in syncing thread")
pass
gevent.sleep(self.sync_interval)
def sync(self, test=False):
pass
class _Registration(object):
"""A service registration."""
def __init__(self, client, form_name, instance_name, data,
interval=3):
self.stopped = Event()
self.client = client
self.form_name = form_name
self.instance_name = instance_name
self.data = data
self.interval = interval
self.gthread = None
def _loop(self):
uri = '/%s/%s' % (self.form_name, self.instance_name)
while not self.stopped.isSet():
response = self.client._request('PUT', uri,
data=json.dumps(self.data))
self.stopped.wait(self.interval)
def start(self):
self.gthread = gevent.spawn(self._loop)
return self
def stop(self, timeout=None):
self.stopped.set()
self.gthread.join(timeout)
class Connection:
def __init__( self, env, connections, handler, timeout ):
self.ack_timeout = timeout
self.ack_done = Event()
self.ack_id = None
self.connections = connections
self.handler = handler
self.server_event_queue = Queue()
self.client_event_queue = Queue()
self._id = uuid().hex
self.env = deepcopydict( env )
self.env['connection'] = self
spawn( self._kill_idle )
spawn\
( self.handler
, self.env
, lambda: self.client_event_queue
, self.server_event_queue.put )
def _check_next( self, confirm_ID ):
result = self.server_event_queue.get()
self.ack_id = confirm_ID
self.current_result.set( (confirm_ID, result ) )
self._kill_idle()
def next( self, confirm_ID, current_ID ):
if confirm_ID == self.ack_id:
self.ack_id = None
self.ack_done.set()
self.current_result = AsyncResult()
spawn( self._check_next, current_ID )
result = self.current_result.get( )
return result
def emit( self, event ):
self.client_event_queue.put( event )
def _kill_idle( self ):
self.ack_done.clear()
self.ack_done.wait( timeout=self.ack_timeout )
if not self.ack_done.isSet():
self.close()
def close( self ):
self.client_event_queue.put( StopIteration )
self.server_event_queue.put( StopIteration )
del self.connections[ self._id ]
class Connection(object):
def __init__(self, env, connections, handler, timeout):
self.ack_timeout = timeout
self.ack_done = Event()
self.ack_id = None
self.connections = connections
self.handler = handler
self.server_event_queue = Queue()
self.client_event_queue = Queue()
self._id = uuid().hex
self.env = deepcopydict(env)
self.env['connection'] = self
spawn(self._kill_idle)
spawn\
( self.handler
, self.env
, lambda: self.client_event_queue
, self.server_event_queue.put
)
def _check_next(self, confirm_ID):
result = self.server_event_queue.get()
self.ack_id = confirm_ID
self.current_result.set((confirm_ID, result))
self._kill_idle()
def next(self, confirm_ID, current_ID):
if confirm_ID == self.ack_id:
self.ack_id = None
self.ack_done.set()
self.current_result = AsyncResult()
spawn(self._check_next, current_ID)
result = self.current_result.get()
return result
def emit(self, event):
self.client_event_queue.put(event)
def _kill_idle(self):
self.ack_done.clear()
self.ack_done.wait(timeout=self.ack_timeout)
if not self.ack_done.isSet():
self.close()
def close(self):
self.client_event_queue.put(StopIteration)
self.server_event_queue.put(StopIteration)
del self.connections[self._id]
class ActionResult(object):
def __init__(self, slave, action, state=PENDING):
self.id_ = id(self)
self.slave = slave
self.action = action
self._state = state
self._text = ""
self.event = Event()
@property
def state(self):
return self._state
@state.setter
def state(self, state):
if state not in (PENDING, RUNNING, SUCCESS, FAILURE):
raise ValueError("Invalid state: %s" % state)
self._state = state
if state in (SUCCESS, FAILURE):
self.event.set()
@property
def text(self):
return self._text
@text.setter
def text(self, text):
self._text = text
def is_done(self):
if self.event.isSet():
return True
else:
return False
def serialize(self, include_requestid=False):
data = {"state": self.state, "text": self.text}
if include_requestid:
data["requestid"] = self.id_
return data
def wait(self, timeout=None):
return self.event.wait(timeout)
class ActionResult(object):
"""Contains basic information about the result of a specific Action."""
def __init__(self, slave, action, state=PENDING,
request_timestamp=0,
start_timestamp=0,
finish_timestamp=0):
self.id_ = id(self)
self.slave = slave
self.action = action
self._state = state
self._text = ""
if not request_timestamp:
request_timestamp = time.time()
self._request_timestamp = request_timestamp
self._start_timestamp = start_timestamp
self._finish_timestamp = finish_timestamp
self.event = Event()
@property
def state(self):
return self._state
@state.setter
def state(self, state):
if state not in (PENDING, RUNNING, SUCCESS, FAILURE):
raise ValueError("Invalid state: %s" % state)
self._state = state
if state in (SUCCESS, FAILURE):
self.event.set()
@property
def text(self):
return self._text
@text.setter
def text(self, text):
self._text = text
@property
def request_timestamp(self):
return self._request_timestamp
@request_timestamp.setter
def request_timestamp(self, timestamp):
self._request_timestamp = timestamp
@property
def start_timestamp(self):
return self._start_timestamp
@start_timestamp.setter
def start_timestamp(self, timestamp):
self._start_timestamp = timestamp
@property
def finish_timestamp(self):
return self._finish_timestamp
@finish_timestamp.setter
def finish_timestamp(self, timestamp):
self._finish_timestamp = timestamp
def is_done(self):
if self.event.isSet():
return True
else:
return False
def to_dict(self, include_requestid=False):
"""Returns the state and text of this ActionResult in a dict. If
include_requestid is True, "requestid" will also be present. Example:
.. code-block:: python
{
"state": 2,
"text": "Great success!",
"request_timestamp": 1392414314,
"start_timestamp": 1392414315,
"finish_timestamp": 1392414316,
"requestid": "234567832"
}
"""
data = {"state": self.state, "text": self.text,
"request_timestamp": self._request_timestamp,
"start_timestamp": self._start_timestamp,
"finish_timestamp": self._finish_timestamp}
if include_requestid:
data["requestid"] = self.id_
return data
def wait(self, timeout=None):
return self.event.wait(timeout)
class AceClient(PVRClient):
ENGINE_TYPE = ('pid', 'torrent')
def __init__(self, vlc_client, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Logger
logger = logging.getLogger('AceClient_init')
self.vlc_client = vlc_client
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException(
"Socket creation error! Ace is not running? " + repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self.vlc_client.stopBroadcast(self.stream_name)
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def _write(self, message):
try:
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def getType(self):
return AceClient.ENGINE_TYPE
def init(self, stream_name, gender=PVRConst.SEX_MALE, age=PVRConst.AGE_18_24, product_key=None, pause_delay=0):
self._product_key = product_key
self._gender = gender
self._age = age
self.stream_name = stream_name
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
logger = logging.getLogger("AceClient_START")
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
errmsg = "START error!"
logger.error(errmsg)
raise AceException(errmsg)
except gevent.Timeout:
errmsg = "START timeout!"
logger.error(errmsg)
raise AceException(errmsg)
return result
def START(self, datatype, value):
'''
Start video method
'''
self._result = AsyncResult()
self._urlresult = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value))
contentinfo = self._getResult()
self._write(AceMessage.request.START(datatype.upper(), value))
self._getResult()
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
self._resumeevent.wait(timeout=timeout)
return
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except urllib2.URLError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
_filename = urllib2.unquote(_contentinfo.get('files')[0][0])
self._result.set(_filename)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
class PriorityQueue(object):
"""A priority queue.
It is greenlet-safe, and offers the ability of changing priorities
and removing arbitrary items.
The queue is implemented as a custom min-heap. The priority is a
mix of a discrete priority level and of the timestamp. The
elements of the queue are QueueItems.
"""
PRIORITY_EXTRA_HIGH = 0
PRIORITY_HIGH = 1
PRIORITY_MEDIUM = 2
PRIORITY_LOW = 3
PRIORITY_EXTRA_LOW = 4
def __init__(self):
"""Create a priority queue."""
# The queue: a min-heap whose elements are of the form
# (priority, timestamp, item), where item is the actual data.
self._queue = []
# Reverse lookup for the items in the queue: a dictionary
# associating the index in the queue to each item.
self._reverse = {}
# Event to signal that there are items in the queue.
self._event = Event()
# Index of the next element that will be added to the queue.
self._next_index = 0
def __len__(self):
return len(self._queue)
def _verify(self):
"""Make sure that the internal state of the queue is consistent.
This is used only for testing.
"""
if len(self._queue) != len(self._reverse):
return False
if len(self._queue) != self.length():
return False
if self.empty() != (self.length() == 0):
return False
if self._event.isSet() == self.empty():
return False
for item, idx in self._reverse.iteritems():
if self._queue[idx].item != item:
return False
return True
def __contains__(self, item):
"""Implement the 'in' operator for an item in the queue.
item (QueueItem): an item to search.
return (bool): True if item is in the queue.
"""
return item in self._reverse
def _swap(self, idx1, idx2):
"""Swap two elements in the queue, keeping their reverse
indices up to date.
idx1 (int): the index of the first element.
idx2 (int): the index of the second element.
"""
self._queue[idx1], self._queue[idx2] = \
self._queue[idx2], self._queue[idx1]
self._reverse[self._queue[idx1].item] = idx1
self._reverse[self._queue[idx2].item] = idx2
def _up_heap(self, idx):
"""Take the element in position idx up in the heap until its
position is the right one.
idx (int): the index of the element to lift.
return (int): the new index of the element.
"""
while idx > 0:
parent = (idx - 1) // 2
if self._queue[idx] < self._queue[parent]:
self._swap(parent, idx)
idx = parent
else:
break
return idx
def _down_heap(self, idx):
"""Take the element in position idx down in the heap until its
position is the right one.
idx (int): the index of the element to lower.
return (int): the new index of the element.
"""
last = len(self._queue) - 1
while 2 * idx + 1 <= last:
child = 2 * idx + 1
if 2 * idx + 2 <= last and \
self._queue[2 * idx + 2] < self._queue[child]:
child = 2 * idx + 2
if self._queue[child] < self._queue[idx]:
self._swap(child, idx)
idx = child
else:
break
return idx
def _updown_heap(self, idx):
"""Perform both operations of up_heap and down_heap on an
element.
idx (int): the index of the element to lift.
return (int): the new index of the element.
"""
idx = self._up_heap(idx)
return self._down_heap(idx)
def push(self, item, priority=None, timestamp=None):
"""Push an item in the queue. If timestamp is not specified,
uses the current time.
item (QueueItem): the item to add to the queue.
priority (int|None): the priority of the item, or None for
medium priority.
timestamp (datetime|None): the time of the submission, or None
to use now.
return (bool): false if the element was already in the queue
and was not pushed again, true otherwise..
"""
if item in self._reverse:
return False
if priority is None:
priority = PriorityQueue.PRIORITY_MEDIUM
if timestamp is None:
timestamp = make_datetime()
index = self._next_index
self._next_index += 1
self._queue.append(QueueEntry(item, priority, timestamp, index))
last = len(self._queue) - 1
self._reverse[item] = last
self._up_heap(last)
# Signal to listener greenlets that there might be something.
self._event.set()
return True
def top(self, wait=False):
"""Return the first element in the queue without extracting it.
wait (bool): if True, block until an element is present.
return (QueueEntry): first element in the queue.
raise (LookupError): on empty queue if wait was false.
"""
if not self.empty():
return self._queue[0]
else:
if not wait:
raise LookupError("Empty queue.")
else:
while True:
if self.empty():
self._event.wait()
continue
return self._queue[0]
def pop(self, wait=False):
"""Extract (and return) the first element in the queue.
wait (bool): if True, block until an element is present.
return (QueueEntry): first element in the queue.
raise (LookupError): on empty queue, if wait was false.
"""
top = self.top(wait)
last = len(self._queue) - 1
self._swap(0, last)
del self._reverse[top.item]
del self._queue[last]
# last is 0 when the queue becomes empty.
if last > 0:
self._down_heap(0)
else:
# Signal that there is nothing left for listeners.
self._event.clear()
return top
def remove(self, item):
"""Remove an item from the queue. Raise a KeyError if not present.
item (QueueItem): the item to remove.
return (QueueEntry): the complete entry removed.
raise (KeyError): if item not present.
"""
pos = self._reverse[item]
entry = self._queue[pos]
last = len(self._queue) - 1
self._swap(pos, last)
del self._reverse[item]
del self._queue[last]
if pos != last:
self._updown_heap(pos)
if self.empty():
self._event.clear()
return entry
def set_priority(self, item, priority):
"""Change the priority of an item inside the queue. Raises an
exception if the item is not in the queue.
item (QueueItem): the item whose priority needs to change.
priority (int): the new priority.
raise (LookupError): if item not present.
"""
pos = self._reverse[item]
self._queue[pos].priority = priority
self._updown_heap(pos)
def length(self):
"""Return the number of elements in the queue.
return (int): length of the queue
"""
return len(self._queue)
def empty(self):
"""Return if the queue is empty.
return (bool): is the queue empty?
"""
return self.length() == 0
def get_status(self):
"""Return the content of the queue. Note that the order may be not
correct, but the first element is the one at the top.
return ([QueueEntry]): a list of entries containing the
representation of the item, the priority and the
timestamp.
"""
return [{'item': entry.item.to_dict(),
'priority': entry.priority,
'timestamp': make_timestamp(entry.timestamp)}
for entry in self._queue]
class AceClient(object):
def __init__(self, acehostslist, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = AceConfig.videoseekback
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._streamReaderConnection = None
self._streamReaderState = None
self._lock = threading.Condition(threading.Lock())
self._streamReaderQueue = gevent.queue.Queue(
maxsize=AceConfig.readcachesize) # Ring buffer
self._engine_version_code = 0
# Logger
logger = logging.getLogger('AceClientimport tracebacknt_init')
# Try to connect AceStream engine
for AceEngine in acehostslist:
try:
self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1],
connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[
0], AceEngine[1], AceEngine[2]
logger.debug('Successfully connected to AceStream on %s:%d' %
(AceEngine[0], AceEngine[1]))
break
except:
logger.debug('The are no alive AceStream on %s:%d' %
(AceEngine[0], AceEngine[1]))
pass
# Spawning recvData greenlet
if self._socket:
gevent.spawn(self._recvData)
gevent.sleep()
else:
logger.error('The are no alive AceStream Engines found')
return
def destroy(self):
'''
AceClient Destructor
'''
logger = logging.getLogger('AceClient_destroy') # Logger
if self._shuttingDown.isSet():
return # Already in the middle of destroying
self._resumeevent.set(
) # We should resume video to prevent read greenlet deadlock
self._urlresult.set() # And to prevent getUrl deadlock
# Trying to disconnect
try:
logger.debug('Destroying AceStream client.....')
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
pass # Ignore exceptions on destroy
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger('AceClient_write')
logger.debug('>>> %s' % message)
self._socket.write('%s\r\n' % message)
except EOFError as e:
raise AceException('Write error! %s' % repr(e))
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_25_34,
product_key=AceConfig.acekey):
self._product_key = product_key
self._gender = gender
self._age = age
self._seekback = AceConfig.videoseekback
self._started_again = False
logger = logging.getLogger('AceClient_aceInit')
self._write(AceMessage.request.HELLO) # Sending HELLOBG
if not self._authevent.wait(self._resulttimeout):
errmsg = 'Authentication timeout during AceEngine init. Wrong key?' # HELLOTS not resived from engine
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = 'Authentication error during AceEngine init. Wrong key?'
logger.error(errmsg)
raise AceException(errmsg)
return
# Display download_stopped massage
if self._engine_version_code >= 3003600:
self._write(AceMessage.request.SETOPTIONS)
def _getResult(self):
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException('Result not received from %s:%s' %
(AceConfig.acehost, AceConfig.aceAPIport))
except gevent.Timeout:
raise AceException('gevent_Timeout')
return result
def START(self, datatype, value, stream_type):
'''
Start video method
'''
if stream_type == 'hls' and self._engine_version_code >= 3010500:
stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \
+ ' transcode_mp3=' + str(AceConfig.transcode_mp3) \
+ ' transcode_ac3=' + str(AceConfig.transcode_ac3) \
+ ' preferred_audio_language=' + AceConfig.preferred_audio_language
else:
stream_type = 'output_format=http'
self._urlresult = AsyncResult()
self._write(
AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, params):
self._result = AsyncResult()
self._write(
AceMessage.request.LOADASYNC(
datatype.upper(),
random.randint(1, AceConfig.maxconns * 10000), params))
return self._getResult()
def GETCONTENTINFO(self, datatype, value):
paramsdict = {
datatype: value,
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
return self.LOADASYNC(datatype, paramsdict)
def GETCID(self, datatype, url):
contentinfo = self.GETCONTENTINFO(datatype, url)
self._cidresult = AsyncResult()
self._write(
AceMessage.request.GETCID(contentinfo.get('checksum'),
contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def getUrl(self, timeout=30):
logger = logging.getLogger('AceClient_getURL') # Logger
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = 'Engine response time exceeded. GetURL timeout!'
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger('StreamReader')
logger.debug('Open video stream: %s' % url)
self._streamReaderState = 1
transcoder = None
if 'range' in req_headers: del req_headers['range']
logger.debug('Get headers from client: %s' % req_headers)
with requests.get(url,
headers=req_headers,
stream=True,
timeout=(5, None)) as self._streamReaderConnection:
try:
if self._streamReaderConnection.status_code not in (200, 206):
logger.error('Failed to open video stream %s' % url)
return None
if url.endswith('.m3u8'):
self._streamReaderConnection.headers = {
'Content-Type': 'application/octet-stream',
'Connection': 'Keep-Alive',
'Keep-Alive': 'timeout=15, max=100'
}
popen_params = {
"bufsize": AceConfig.readchunksize,
"stdout": PIPE,
"stderr": None,
"shell": False
}
if AceConfig.osplatform == 'Windows':
ffmpeg_cmd = 'ffmpeg.exe '
CREATE_NO_WINDOW = 0x08000000
CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
popen_params.update(creationflags=CREATE_NO_WINDOW
| DETACHED_PROCESS
| CREATE_NEW_PROCESS_GROUP)
else:
ffmpeg_cmd = 'ffmpeg '
ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url
transcoder = Popen(ffmpeg_cmd.split(), **popen_params)
out = transcoder.stdout
logger.warning(
'HLS stream detected. Ffmpeg transcoding started')
else:
out = self._streamReaderConnection.raw
except requests.exceptions.RequestException:
logger.error('Failed to open video stream %s' % url)
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
else:
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
self.play_event()
while 1:
self.getPlayEvent(
) # Wait for PlayEvent (stop/resume sending data from AceEngine to streamReaderQueue)
clients = counter.getClients(cid)
try:
data = out.read(AceConfig.readchunksize)
except:
data = None
if data is not None and clients:
if self._streamReaderQueue.full():
self._streamReaderQueue.get()
self._streamReaderQueue.put(data)
for c in clients:
try:
c.queue.put(data, timeout=5)
except gevent.queue.Full: #Queue.Full client does not read data from buffer until 5sec - disconnect it
if len(clients) > 1:
logger.debug('Disconnecting client: %s' %
c.handler.clientip)
c.destroy()
elif counter.count(cid) == 0:
logger.debug(
'All clients disconnected - broadcast stoped')
break
else:
logger.warning('No data received - broadcast stoped')
counter.deleteAll(cid)
break
finally:
with self._lock:
self._streamReaderState = None
self._lock.notifyAll()
if transcoder:
try:
transcoder.kill()
logger.warning('Ffmpeg transcoding stoped')
except:
pass
def closeStreamReader(self):
logger = logging.getLogger('StreamReader')
c = self._streamReaderConnection
if c:
logger.debug('Close video stream: %s' % c.url)
c.close()
self._streamReaderConnection = None
self._streamReaderQueue.queue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
# EVENTS from client to AceEngine
def play_event(self):
self._write(AceMessage.request.PLAYEVENT)
def pause_event(self):
self._write(AceMessage.request.PAUSEEVENT)
def stop_event(self):
self._write(AceMessage.request.STOPEVENT)
# END EVENTS
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while 1:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until('\r\n').strip()
logger.debug(
'<<< %s' %
requests.compat.unquote(self._recvbuffer).decode('utf8'))
except:
# If something happened during read, abandon reader.
logger.error('Exception at socket read. AceClient destroyed')
if not self._shuttingDown.isSet(): self._shuttingDown.set()
return
else:
# Parsing everything only if the string is not empty
# HELLOTS
if self._recvbuffer.startswith(AceMessage.response.HELLO):
try:
params = {
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
}
except:
logger.error("Can't parse HELLOTS")
params = {}
if 'version_code' in params:
self._engine_version_code = params['version_code']
if 'key' in params:
self._write(
AceMessage.request.READY_key(
params['key'], self._product_key))
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
# NOTREADY
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
self._auth = None
self._authevent.set()
logger.error('AceEngine is not ready. Wrong auth?')
# AUTH
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
self._write(
AceMessage.request.USERDATA(
self._gender, self._age))
except:
pass
self._authevent.set()
# GETUSERDATA
elif self._recvbuffer.startswith(
AceMessage.response.GETUSERDATA):
raise AceException('You should init me first!')
# LOADRESP
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
_contentinfo = json.loads(
requests.compat.unquote(' '.join(
self._recvbuffer.split()[2:])).decode('utf8'))
if _contentinfo.get('status') == 100:
logger.error(
'LOADASYNC returned error with message: %s' %
_contentinfo.get('message'))
self._result.set(False)
else:
self._result.set(_contentinfo)
# START
elif self._recvbuffer.startswith(AceMessage.response.START):
if not self._seekback or self._started_again or not self._recvbuffer.endswith(
' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._urlresult.set(self._recvbuffer.split()[1])
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug('START received. Waiting for %s.' %
AceMessage.response.LIVEPOS)
# STATE
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
if self._state in ('0', '1'):
self._result.set(True) #idle, starting
elif self._state == '6':
self._result.set(False) # error
# STATUS
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._status = self._recvbuffer.split()[1].split(';')
if 'main:err' in set(
self._status): # main:err;error_id;error_message
logger.error('%s with message %s' %
(self._status[0], self._status[2]))
self._result.set_exception(
AceException('%s with message %s' %
(self._status[0], self._status[2])))
self._urlresult.set_exception(
AceException('%s with message %s' %
(self._status[0], self._status[2])))
# LIVEPOS
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
if self._seekback and not self._started_again:
try:
params = {
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
}
self._write(
AceMessage.request.LIVESEEK(
int(params['last']) - self._seekback))
logger.debug('Seeking back.....')
self._started_again = True
except:
logger.error("Can't parse %s" %
AceMessage.response.LIVEPOS)
# CID
elif self._recvbuffer.startswith('##') or len(
self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
#DOWNLOADSTOP
elif self._recvbuffer.startswith(
AceMessage.response.DOWNLOADSTOP):
pass
# INFO
elif self._recvbuffer.startswith(AceMessage.response.INFO):
pass
# SHOWURL
elif self._recvbuffer.startswith(AceMessage.response.SHOWURL):
pass
# CANSAVE
elif self._recvbuffer.startswith(AceMessage.response.CANSAVE):
pass
# PAUSE
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
self.pause_event()
self._resumeevent.clear()
# RESUME
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
self.play_event()
self._resumeevent.set()
# STOP
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
# SHUTDOWN
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
self._socket.get_socket().shutdown(SHUT_WR)
self._recvbuffer = self._socket.read_all()
self._socket.close()
logger.debug('AceClient destroyed')
return
class AceClient(object):
def __init__(self, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = 0
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0
# Logger
logger = logging.getLogger('AceClieimport tracebacknt_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException("Socket creation error! Ace is not running? " +
repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger("AceClient_write")
logger.debug(message)
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_18_24,
product_key=None,
pause_delay=0,
seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException("Result not received")
except gevent.Timeout:
raise AceException("Timeout")
return result
def START(self, datatype, value):
'''
Start video method
'''
stream_type = 'output_format=http' if self._engine_version_code >= 3010500 and not AceConfig.vlcuse else ''
self._urlresult = AsyncResult()
self._write(
AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state is not None and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, url):
self._result = AsyncResult()
self._write(
AceMessage.request.LOADASYNC(datatype.upper(), 0, {'url': url}))
return self._getResult()
def GETCID(self, datatype, url):
contentinfo = self.LOADASYNC(datatype, url)
self._cidresult = AsyncResult()
self._write(
AceMessage.request.GETCID(contentinfo.get('checksum'),
contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def GETCONTENTINFO(self, datatype, url):
contentinfo = self.LOADASYNC(datatype, url)
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter):
logger = logging.getLogger("StreamReader")
self._streamReaderState = 1
logger.debug("Opening video stream: %s" % url)
try:
connection = self._streamReaderConnection = urllib2.urlopen(url)
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return
if connection.getcode() != 200:
logger.error("Failed to open video stream %s" % connection)
return
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
while True:
data = None
clients = counter.getClients(cid)
try:
data = connection.read(AceConfig.readchunksize)
except:
break
if data and clients:
with self._lock:
if len(self._streamReaderQueue
) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" %
str(c))
c.destroy()
elif not clients:
logger.debug(
"All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except urllib2.URLError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClients(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger("StreamReader")
c = self._streamReaderConnection
if c:
self._streamReaderConnection = None
c.close()
logger.debug("Video stream closed")
self._streamReaderQueue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
# logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'version_code=' in self._recvbuffer:
v = self._recvbuffer.find('version_code=')
self._engine_version_code = int(
self._recvbuffer[v + 13:v + 20])
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14]
try:
self._write(
AceMessage.request.READY_key(
self._request_key, self._product_key))
except urllib2.URLError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error(
"LOADASYNC returned error with message: %s" %
_contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
self._result.set(_contentinfo)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or self._started_again or not self._recvbuffer.endswith(
' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seeback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug("START received. Waiting for %s." %
AceMessage.response.LIVEPOS)
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(
self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(
AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
if self._seekback and not self._started_again:
self._write(AceMessage.request.SEEK(str(int(self._position_last) - \
self._seekback)))
logger.debug('Seeking back')
self._started_again = True
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(
';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(self._status + ' with message ' +
self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' +
self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' +
self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._status == 'main:idle':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
elif self._recvbuffer.startswith('##') or len(
self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
logger.debug("CID: %s" % self._recvbuffer)
def sendHeadersPT(self, client, code, headers):
client.handler.send_response(code)
for key, value in headers.items():
client.handler.send_header(key, value)
client.handler.end_headers()
def openStreamReaderPT(self, url, req_headers):
logger = logging.getLogger("openStreamReaderPT")
logger.debug("Opening video stream: %s" % url)
logger.debug("headers: %s" % req_headers)
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return None, None, None
request = urllib2.Request(url, headers=req_headers)
connection = self._streamReaderConnection = urllib2.urlopen(
request, timeout=120)
code = connection.getcode()
if code not in (200, 206):
logger.error("Failed to open video stream %s" % connection)
return None, None, None
FORWARD_HEADERS = [
'Content-Range',
'Connection',
'Keep-Alive',
'Content-Type',
'Accept-Ranges',
'X-Content-Duration',
'Content-Length',
]
SKIP_HEADERS = ['Server', 'Date']
response_headers = {}
for k in connection.info().headers:
if k.split(':')[0] not in (FORWARD_HEADERS + SKIP_HEADERS):
logger.debug('NEW HEADERS: %s' % k.split(':')[0])
for h in FORWARD_HEADERS:
if connection.info().getheader(h):
response_headers[h] = connection.info().getheader(h)
# self.connection.send_header(h, connection.info().getheader(h))
logger.debug('key=%s value=%s' %
(h, connection.info().getheader(h)))
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
return connection, code, response_headers
def startStreamReaderPT(self, url, cid, counter, req_headers=None):
logger = logging.getLogger("StreamReaderPT")
self._streamReaderState = 1
# current_req_headers = req_headers
try:
while True:
data = None
clients = counter.getClientsPT(cid)
if not req_headers == self.req_headers:
self.req_headers = req_headers
connection, code, resp_headers = self.openStreamReaderPT(
url, req_headers)
if not connection:
return
for c in clients:
try:
c.headers_sent
except:
self.sendHeadersPT(c, code, resp_headers)
c.headers_sent = True
# logger.debug("i")
try:
data = connection.read(AceConfig.readchunksize)
except:
break
# logger.debug("d: %s c:%s" % (data, clients))
if data and clients:
with self._lock:
if len(self._streamReaderQueue
) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" %
str(c))
c.destroy()
elif not clients:
logger.debug(
"All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except urllib2.URLError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClientsPT(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
class AceClient(object):
def __init__(self, acehost, aceAPIport, aceHTTPport, acehostslist, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = AceConfig.videoseekback
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0;
# Logger
logger = logging.getLogger('AceClientimport tracebacknt_init')
# Try to connect AceStream engine
try:
self._socket = telnetlib.Telnet(acehost, aceAPIport, connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = acehost, aceAPIport, aceHTTPport
logger.debug("Successfully connected to AceStream on %s:%d" % (acehost, aceAPIport))
except:
for AceEngine in acehostslist:
try:
self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[0], AceEngine[1], AceEngine[2]
logger.debug("Successfully connected to AceStream on %s:%d" % (AceEngine[0], AceEngine[1]))
break
except:
logger.debug("The are no alive AceStream on %s:%d" % (AceEngine[0], AceEngine[1]))
pass
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger("AceClient_write")
logger.debug(message)
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException("Result not received")
except gevent.Timeout:
raise AceException("Timeout")
return result
def START(self, datatype, value):
'''
Start video method
'''
if self._engine_version_code >= 3010500 and AceConfig.vlcuse:
stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \
+ ' transcode_mp3=' + str(AceConfig.transcode_mp3) \
+ ' transcode_ac3=' + str(AceConfig.transcode_ac3) \
+ ' preferred_audio_language=' + AceConfig.preferred_audio_language
else:
stream_type = 'output_format=http'
self._urlresult = AsyncResult()
self._write(AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state is not None and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, value):
self._result = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), random.randint(1, AceConfig.maxconns * 10000), value))
return self._getResult()
def GETCONTENTINFO(self, datatype, value):
dict = {'torrent': 'url', 'infohash':'infohash', 'raw':'data', 'pid':'content_id'}
self.paramsdict={dict[datatype]:value,'developer_id':'0','affiliate_id':'0','zone_id':'0'}
contentinfo = self.LOADASYNC(datatype, self.paramsdict)
return contentinfo
def GETCID(self, datatype, url):
contentinfo = self.GETCONTENTINFO(datatype, url)
self._cidresult = AsyncResult()
self._write(AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def getUrl(self, timeout=30):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "Engine response time exceeded. GetURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger("StreamReader")
logger.debug("Opening video stream: %s" % url)
logger.debug("Get headers from client: %s" % req_headers)
self._streamReaderState = 1
try:
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return None
# Sending client hedaers to AceEngine
if req_headers.has_key('range'):
del req_headers['range']
connection = self._streamReaderConnection = requests.get(url, headers=req_headers, stream=True)
if connection.status_code not in (200, 206):
logger.error("Failed to open video stream %s" % url)
return None
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
while True:
data = None
clients = counter.getClients(cid)
try:
data = connection.raw.read(AceConfig.readchunksize)
except:
break;
if data and clients:
with self._lock:
if len(self._streamReaderQueue) == AceConfig.readchunksize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" % str(c))
c.destroy()
elif not clients:
logger.debug("All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except requests.exceptions.ConnectionError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClients(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger("StreamReader")
c = self._streamReaderConnection
if c:
self._streamReaderConnection = None
c.close()
logger.debug("Video stream closed")
self._streamReaderQueue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'version_code=' in self._recvbuffer:
v = self._recvbuffer.find('version_code=')
self._engine_version_code = int(self._recvbuffer[v+13:v+20])
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except requests.exceptions.ConnectionError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
self._result.set(_contentinfo)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or self._started_again or not self._recvbuffer.endswith(' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS)
# STOP
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
# SHUTDOWN
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
# AUTH
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
# GETUSERDATA
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
# LIVEPOS
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
if self._seekback and not self._started_again:
self._write(AceMessage.request.LIVESEEK(str(int(self._position_last) - self._seekback)))
logger.debug('Seeking back')
self._started_again = True
# DOWNLOADSTOP
elif self._recvbuffer.startswith(AceMessage.response.DOWNLOADSTOP):
self._state = self._recvbuffer.split()[1]
# STATE
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
# STATUS
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._status == 'main:idle':
self._result.set(True)
# PAUSE
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
# RESUME
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep() # PAUSE/RESUME delay
self._resumeevent.set()
# CID
elif self._recvbuffer.startswith('##') or len(self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
logger.debug("CID: %s" %self._recvbuffer)
class TestMemoryCache(unittest.TestCase):
"""
Test description
"""
currentTempDir = ""
def setUp(self):
"""
Setup
"""
# Reset counters
Meters.reset()
# Clear
self.mem_cache = None
self.callback_call = 0
self.callback_evicted = 0
self.callback_return = False
self.evict_count = 0
self.evict_last_key = None
self.evict_last_value = None
def tearDown(self):
"""
Stop
"""
if self.mem_cache:
logger.warning("Stopping mem_cache")
self.mem_cache.stop_cache()
self.mem_cache = None
def watchdog_callback(self, evicted_count):
"""
Callback
:param evicted_count: Evicted count
:return: True if reschedule the callback, False otherwise
"""
logger.info("Called, evicted_count=%s", evicted_count)
self.callback_call += 1
self.callback_evicted += evicted_count
return self.callback_return
def eviction_callback(self, k, v):
"""
Eviction callback
:param k: key
:param v: value
"""
self.evict_count += 1
self.evict_last_key = k
self.evict_last_value = v
def test_start_stop(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache()
self.assertTrue(self.mem_cache._is_started)
# Stop
self.mem_cache.stop_cache()
self.assertFalse(self.mem_cache._is_started)
# Start
self.mem_cache.start_cache()
self.assertTrue(self.mem_cache._is_started)
# Stop
self.mem_cache.stop_cache()
self.assertFalse(self.mem_cache._is_started)
# Over
self.mem_cache = None
def test_size_handling(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache()
# Put
self.mem_cache.put("A", b"A1", 60000)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 2)
# Put again, not the same data
self.mem_cache.put("A", b"A11", 60000)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 3)
# Put again, not the same data
self.mem_cache.put("A", b"A", 60000)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 1)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_init_no_stat(self):
"""
Test.
"""
# Alloc
m = MemoryCache()
self.assertIsNotNone(m)
def test_basic(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache()
# Get : must return nothing
o = self.mem_cache.get("not_found")
self.assertIsNone(o)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
o = self.mem_cache.get("keyA")
self.assertEqual(o, b"valA")
o = self.mem_cache.get_raw("keyA")
self.assertIsNotNone(o)
self.assertIsInstance(o, tuple)
self.assertEqual(o[1], b"valA")
self.assertLessEqual(o[0] - SolBase.mscurrent(), 60000)
logger.info("TTL approx=%s", o[0] - SolBase.mscurrent())
# Put with lower TTL : TTL MUST BE UPDATED
self.mem_cache.put("keyA", b"valA", 30000)
o = self.mem_cache.get("keyA")
self.assertEqual(o, b"valA")
o = self.mem_cache.get_raw("keyA")
self.assertIsNotNone(o)
self.assertIsInstance(o, tuple)
self.assertEqual(o[1], b"valA")
self.assertLessEqual(o[0] - SolBase.mscurrent(), 30000)
logger.info("TTL approx=%s", o[0] - SolBase.mscurrent())
# Str put : must now be ok
self.mem_cache.put("toto_str", "str_val", 1000)
self.assertEqual(self.mem_cache.get("toto_str"), "str_val")
# Non bytes,str injection (int) : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.mem_cache.put("toto", 12, 1000)
self.fail("Must fail")
except:
pass
# Non bytes injection : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.mem_cache.put("toto", u"unicode_buffer", 1000)
self.fail("Must fail")
except:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.mem_cache.put(999, b"value", 60000)
self.fail("Put a key as non bytes,str MUST fail")
except Exception:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.mem_cache.remove(999)
self.fail("Remove a key as non bytes,str MUST fail")
except Exception:
pass
# Put/Remove
self.mem_cache.put("todel", b"value", 60000)
self.assertEqual(self.mem_cache.get("todel"), b"value")
self.mem_cache.remove("todel")
self.assertEqual(self.mem_cache.get("todel"), None)
# Put
self.mem_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz",
60000)
self.assertEqual(
self.mem_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"), b"zzz")
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic_eviction_max_capacity_lru(self):
"""
Test
:return:
"""
# Alloc
self.mem_cache = MemoryCache(max_item=3,
cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 60000)
self.mem_cache.put("keyC", b"valC", 60000)
# Use A and C => B becomes the older to be used
self.mem_cache.get("keyA")
self.mem_cache.get("keyC")
# We are maxed (3 items)
# Add D => B must be kicked
self.mem_cache.put("keyD", b"valD", 60000)
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
self.assertEqual(self.mem_cache.get("keyC"), b"valC")
self.assertEqual(self.mem_cache.get("keyD"), b"valD")
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertEqual(self.evict_count, 1)
self.assertEqual(self.evict_last_key, "keyB")
self.assertEqual(self.evict_last_value, b"valB")
self.assertEqual(Meters.aig("mcs.cache_evict_lru_put"), 1)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic_eviction_lru(self):
"""
Test
:return:
"""
# Alloc
self.mem_cache = MemoryCache(cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 60000)
self.mem_cache.put("keyC", b"valC", 60000)
# Evict LRU
self.mem_cache._evict_key_lru()
# A was put first => must be kicked
self.assertIsNone(self.mem_cache.get("keyA"))
self.assertEqual(self.mem_cache.get("keyB"), b"valB")
self.assertEqual(self.mem_cache.get("keyC"), b"valC")
self.assertEqual(self.evict_count, 1)
self.assertEqual(self.evict_last_key, "keyA")
self.assertEqual(self.evict_last_value, b"valA")
# Evict LRU
self.mem_cache._evict_key_lru()
# B was put first => must be kicked
self.assertIsNone(self.mem_cache.get("keyA"))
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertEqual(self.mem_cache.get("keyC"), b"valC")
self.assertEqual(self.evict_count, 2)
self.assertEqual(self.evict_last_key, "keyB")
self.assertEqual(self.evict_last_value, b"valB")
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic_eviction_with_get_lru(self):
"""
Test
:return:
"""
# Alloc
self.mem_cache = MemoryCache(cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 60000)
# A was put first => must be kicked, but we GET IT => B must be kicked
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
# Evict LRU
self.mem_cache._evict_key_lru()
# A was used more recently => B must be kicked
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
self.assertEqual(self.evict_count, 1)
self.assertEqual(self.evict_last_key, "keyB")
self.assertEqual(self.evict_last_value, b"valB")
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic_eviction_with_get_ttl(self):
"""
Test
:return:
"""
# Alloc
self.mem_cache = MemoryCache(cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 500)
logger.info("ms cur=%s", SolBase.mscurrent())
logger.info("A : %s", self.mem_cache.get_raw("keyA"))
logger.info("B : %s", self.mem_cache.get_raw("keyB"))
# Wait a bit
SolBase.sleep(600)
logger.info("ms after sleep=%s", SolBase.mscurrent())
# A : must be present
# B : must be evicted (TTL elapsed)
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertEqual(self.evict_count, 1)
self.assertEqual(self.evict_last_key, "keyB")
self.assertEqual(self.evict_last_value, b"valB")
self.assertEqual(Meters.aig("mcs.cache_evict_ttl_get"), 1)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic_eviction_with_watchdog_ttl(self):
"""
Test
:return:
"""
# Go
self.mem_cache = MemoryCache(watchdog_interval_ms=1000,
cb_watchdog=self.watchdog_callback,
cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 500)
self.mem_cache.put("keyC", b"valC", 500)
self.mem_cache.put("keyD", b"valD", 60000)
logger.info("ms cur=%s", SolBase.mscurrent())
logger.info("A : %s", self.mem_cache.get_raw("keyA"))
logger.info("B : %s", self.mem_cache.get_raw("keyB"))
logger.info("C : %s", self.mem_cache.get_raw("keyC"))
logger.info("D : %s", self.mem_cache.get_raw("keyD"))
# Wait a bit
ms_start = SolBase.mscurrent()
while SolBase.msdiff(ms_start) < (1000.0 * 2.0):
if self.callback_call > 0:
break
else:
SolBase.sleep(10)
logger.info("ms after wait=%s", SolBase.mscurrent())
logger.info("_hash_key = %s", self.mem_cache._hash_key)
logger.info("_hash_context = %s", self.mem_cache._hash_context)
# A : must be present
# B : must be evicted (TTL elapsed, by watchdog)
self.assertEqual(self.callback_call, 1)
self.assertEqual(self.callback_evicted, 2)
self.assertFalse("valB" in self.mem_cache._hash_key)
self.assertFalse("valB" in self.mem_cache._hash_context)
self.assertFalse("valC" in self.mem_cache._hash_key)
self.assertFalse("valC" in self.mem_cache._hash_context)
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertIsNone(self.mem_cache.get("keyC"))
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
self.assertEqual(self.mem_cache.get("keyD"), b"valD")
self.assertEqual(self.evict_count, 2)
self.assertTrue(self.evict_last_key == "keyB"
or self.evict_last_key == "keyC")
self.assertTrue(self.evict_last_value == b"valB"
or self.evict_last_value == b"valC")
self.assertEqual(Meters.aig("mcs.cache_evict_ttl_watchdog"), 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_watchdog_schedule(self):
"""
Test
:return:
"""
# Continue callback loop
self.callback_return = True
# Go
self.mem_cache = MemoryCache(watchdog_interval_ms=500,
cb_watchdog=self.watchdog_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyD", b"valD", 60000)
# Wait a bit
ms_start = SolBase.mscurrent()
while SolBase.msdiff(ms_start) < (500.0 * 5.0):
if self.callback_call >= 1:
logger.info("Run 1 exit")
break
else:
SolBase.sleep(1)
logger.info("Run 1 done")
self.assertEqual(self.callback_call, 1)
self.assertEqual(self.callback_evicted, 0)
# Wait a bit
ms_start = SolBase.mscurrent()
while SolBase.msdiff(ms_start) < (500.0 * 5.0):
if self.callback_call >= 2:
logger.info("Run 2 exit")
break
else:
SolBase.sleep(1)
logger.info("Run 2 done")
self.assertGreaterEqual(self.callback_call, 2)
self.assertEqual(self.callback_evicted, 0)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
# Wait a bit
SolBase.sleep(500 * 2)
# Nothing must happen
self.assertEqual(self.callback_call, 2)
self.assertEqual(self.callback_evicted, 0)
# ========================
# BENCH
# ========================
def test_bench_greenlet_1_put_100_get_0_max_128000(self):
"""
Test
:return:
"""
self._go_greenlet(1, 100, 0, 128000)
def test_bench_greenlet_16_put_10_get_1_max_50(self):
"""
Test
:return:
"""
self._go_greenlet(16, 1, 1, 128000, max_item=50)
def test_bench_greenlet_16_put_10_get_1_maxsize_300bytes(self):
"""
Test
:return:
"""
self._go_greenlet(
16,
1,
1,
128000,
watchdog_interval_ms=500,
max_item=50,
max_bytes=300,
max_single_item_bytes=50,
purge_min_bytes=20,
purge_min_count=5,
)
def test_bench_greenlet_1_put_1_get_100_max_128000(self):
"""
Test
:return:
"""
self._go_greenlet(1, 1, 100, 128000)
def test_bench_greenlet_128_put_10_get_10_max_128000(self):
"""
Test
:return:
"""
self._go_greenlet(128, 10, 10, 128000)
def test_bench_greenlet_128_put_10_get_10_maxsize_10000(self):
"""
Test
:return:
"""
self._go_greenlet(
128,
10,
10,
128000,
watchdog_interval_ms=500,
max_item=sys.maxsize,
max_bytes=10000,
max_single_item_bytes=50,
purge_min_bytes=int(10000 / 2),
purge_min_count=5000,
)
def _go_greenlet(self,
greenlet_count,
put_count,
get_count,
bench_item_count,
watchdog_interval_ms=60000,
max_item=128000,
max_bytes=32 * 1024 * 1024,
max_single_item_bytes=1 * 1024 * 1024,
purge_min_bytes=8 * 1024 * 1024,
purge_min_count=1000):
"""
Doc
:param greenlet_count: greenlet_count
:param put_count: put_count
:param get_count: get_count
:param bench_item_count: bench_item_count
:param watchdog_interval_ms: watchdog_interval_ms
:param max_item: max_item
:param max_bytes: max_bytes
:param max_single_item_bytes: max_single_item_bytes
:param purge_min_bytes: purge_min_bytes
:param purge_min_count: purge_min_count
"""
g_event = None
g_array = None
try:
# Settings
g_count = greenlet_count
g_ms = 10000
# Continue callback loop
self.callback_return = True
# Go
self.mem_cache = MemoryCache(
watchdog_interval_ms=watchdog_interval_ms,
max_item=max_item,
max_bytes=max_bytes,
max_single_item_bytes=max_single_item_bytes,
purge_min_bytes=purge_min_bytes,
purge_min_count=purge_min_count)
# Item count
self.bench_item_count = bench_item_count
self.bench_put_weight = put_count
self.bench_get_weight = get_count
self.bench_ttl_min_ms = 1000
self.bench_ttl_max_ms = int(g_ms / 2)
# Go
self.run_event = Event()
self.exception_raised = 0
self.open_count = 0
self.thread_running = AtomicIntSafe()
self.thread_running_ok = AtomicIntSafe()
# Item per greenlet
item_per_greenlet = self.bench_item_count / g_count
# Signal
self.gorun_event = Event()
# Alloc greenlet
g_array = list()
g_event = list()
for _ in range(0, g_count):
greenlet = Greenlet()
g_array.append(greenlet)
g_event.append(Event())
# Run them
cur_idx = 0
for idx in range(0, len(g_array)):
greenlet = g_array[idx]
event = g_event[idx]
greenlet.spawn(self._run_cache_bench, event, cur_idx,
cur_idx + item_per_greenlet)
cur_idx += item_per_greenlet
SolBase.sleep(0)
# Signal
self.gorun_event.set()
# Wait a bit
dt = SolBase.mscurrent()
while SolBase.msdiff(dt) < g_ms:
SolBase.sleep(500)
# Stat
ms = SolBase.msdiff(dt)
sec = float(ms / 1000.0)
total_put = Meters.aig("mcs.cache_put")
per_sec_put = round(float(total_put) / sec, 2)
total_get = Meters.aig("mcs.cache_get_hit") + Meters.aig(
"mcs.cache_get_miss")
per_sec_get = round(float(total_get) / sec, 2)
logger.info(
"Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s",
self.open_count, self.thread_running.get(),
self.thread_running_ok.get(), per_sec_put, per_sec_get,
self.mem_cache)
self.assertEqual(self.exception_raised, 0)
# Over, signal
logger.info("Signaling, count=%s", self.open_count)
self.run_event.set()
# Wait
for g in g_event:
g.wait(30.0)
self.assertTrue(g.isSet())
g_event = None
g_array = None
# Log
Meters.write_to_logger()
finally:
self.run_event.set()
if g_event:
for g in g_event:
g.set()
if g_array:
for g in g_array:
g.kill()
if self.mem_cache:
max_count = 0
total_size = 0
i = 0
for (k, v) in self.mem_cache._hash_key.items():
i += 1
total_size += len(k) + len(v[1])
if i < max_count:
logger.info("%s => %s", k, v)
self.assertEqual(total_size,
self.mem_cache._current_data_bytes.get())
self.mem_cache.stop_cache()
self.mem_cache = None
def _run_cache_bench(self, event, idx_min, idx_max):
"""
Run
:param idx_min: Index min
:param idx_max: Index max
"""
idx_max -= 1
# Wait
self.gorun_event.wait()
# Go
cur_count = 0
logger.debug("Entering now, idx_min=%s, idx_max=%s", idx_min, idx_max)
self.thread_running.increment()
self.thread_running_ok.increment()
try:
while not self.run_event.isSet():
cur_count += 1
try:
cur_item = random.randint(idx_min, idx_max)
s_cur_item = "%s" % cur_item
b_cur_item = SolBase.unicode_to_binary(s_cur_item, "utf-8")
cur_ttl = random.randint(self.bench_ttl_min_ms,
self.bench_ttl_max_ms)
for _ in range(0, self.bench_put_weight):
self.mem_cache.put(s_cur_item, b_cur_item, cur_ttl)
SolBase.sleep(0)
for _ in range(0, self.bench_get_weight):
v = self.mem_cache.get(s_cur_item)
if v:
self.assertEqual(v, b_cur_item)
SolBase.sleep(0)
except Exception as e:
self.exception_raised += 1
logger.warning("Ex=%s", SolBase.extostr(e))
self.thread_running_ok.increment(-1)
return
finally:
pass
finally:
self.assertGreater(cur_count, 0)
logger.debug("Exiting")
event.set()
self.thread_running.increment(-1)
# =====================================
# SIZE STUFF
# =====================================
def test_size_limit_on_data_size_first(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(
max_bytes=5 * 10 * 2,
max_single_item_bytes=6 * 2,
purge_min_bytes=5 * 5 * 2,
purge_min_count=2,
max_item=sys.maxsize,
)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 10 * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted AT least :
# 5*5 + 5 bytes => 5 items + the item added => 6 items
# 2 items minimum
# So 6 items : 10 to 15 must be evicted
logger.info("Hash = %s", self.mem_cache._hash_key)
for i in range(10, 16):
self.assertIsNone(self.mem_cache.get("key" + str(i)))
for i in range(16, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 5)
self.assertEqual(len(self.mem_cache._hash_context), 5)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 5 * 2)
# Try add a big one this time : must not be done (over limit)
self.mem_cache.put("BIGDATA", b"aaaaaaaaaaaaaaaaaaaa", 60000)
self.assertIsNone(self.mem_cache.get("BIGDATA"))
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_size_limit_on_min_item_to_evict_first(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(
max_bytes=5 * 10 * 2,
max_single_item_bytes=6 * 2,
purge_min_bytes=1 * 5,
purge_min_count=6,
max_item=sys.maxsize,
)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted AT least :
# 1*5 + 5 bytes => 1 items + the item added => 2 items
# 6 items minimum
# So : 10 to 15 must be evicted
logger.info("Hash = %s", self.mem_cache._hash_key)
for i in range(10, 16):
self.assertIsNone(self.mem_cache.get("key" + str(i)))
for i in range(16, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 5)
self.assertEqual(len(self.mem_cache._hash_context), 5)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Try add a big one this time : must not be done (over limit)
self.mem_cache.put("BIGDATA", b"aaaaaaaaaaaaaaaaaaaa", 60000)
self.assertIsNone(self.mem_cache.get("BIGDATA"))
self.mem_cache.put("aaaaaaaaaaaaaaaaaaaa", b"BIGKEY", 60000)
self.assertIsNone(self.mem_cache.get("aaaaaaaaaaaaaaaaaaaa"))
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_size_limit_cache_clear(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(
max_bytes=5 * 10 * 2,
max_single_item_bytes=6 * 2,
purge_min_bytes=1 * 5,
purge_min_count=100,
max_item=sys.maxsize,
)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted AT least :
# 1*5 + 5 bytes => 1 items + the item added => 2 items
# 100 items minimum
# So : All items must be kicked, will remain only the new one
logger.info("Hash = %s", self.mem_cache._hash_key)
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 1)
self.assertEqual(len(self.mem_cache._hash_context), 1)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_size_limit_max_item_count(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(max_bytes=sys.maxsize,
max_single_item_bytes=6 * 2,
purge_min_bytes=1 * 5,
purge_min_count=0,
max_item=10)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted only first added
logger.info("Hash = %s", self.mem_cache._hash_key)
self.assertIsNone(self.mem_cache.get("key" + str(10)))
for i in range(11, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def _remove_client(self, client_id, evt):
"""
Remove a client. Return a TcpServerClientContext upon success,
:param client_id: The client id.
:type client_id: int
:param evt: Event to signal
:type evt: gevent.Event, None
:return The removed TcpServerClientContext or None upon failure.
:rtype None,pysoltcp.tcpserver.clientcontext.TcpServerClientContext.TcpServerClientContext
"""
logger.debug("entering, client_id=%s", client_id)
try:
with self._client_connected_hash_lock:
# Check
if client_id not in self._client_connected_hash:
# Note : This may occurs in some conditions
logger.debug("client_id not hashed, id=%s", client_id)
Meters.aii("tcp.server.client_remove_nothashed")
return None
# Get (direct, we are already in lock)
old_client = self._get_client_fromid(client_id)
# Remove from hashmap
logger.debug("un-hashing, client_id=%s", client_id)
del (self._client_connected_hash[client_id])
# ------------------------------
# Out of lock : call async : BUSINESS
# ------------------------------
try:
local_evt = Event()
g1 = gevent.spawn(self._remove_client_stop_business,
old_client, local_evt)
SolBase.sleep(0)
local_evt.wait(self._tcp_server_config.stop_client_timeout_ms /
1000.0)
if not local_evt.isSet():
# Flush out warning
s = "Greenlet dump, g={0}, frame={1}".format(
g1, ''.join(traceback.format_stack(g1.gr_frame)))
# Cleanup
s = s.replace("\n", " # ")
while s.find(" ") >= 0:
s = s.replace(" ", " ")
# Error logs
logger.error(
"Timeout in _remove_client_stop_business client_id=%s, stack=%s",
client_id, s)
# Kill
g1.kill(block=True)
# Stat
Meters.aii("tcp.server.client_remove_timeout_business")
except Exception as e:
logger.warning(
"Exception in _remove_client_stop_business client_id=%s, ex=%s",
client_id, SolBase.extostr(e))
# ------------------------------
# Out of lock : call async : INTERNAL
# ------------------------------
try:
local_evt = Event()
g2 = gevent.spawn(self._remove_client_stop_internal,
old_client, local_evt)
SolBase.sleep(0)
local_evt.wait(self._tcp_server_config.stop_client_timeout_ms /
1000.0)
if not local_evt.isSet():
# Flush out warning
s = "Greenlet dump, g={0}, frame={1}".format(
g2, ''.join(traceback.format_stack(g2.gr_frame)))
# Cleanup
s = s.replace("\n", " # ")
while s.find(" ") >= 0:
s = s.replace(" ", " ")
# Error logs
logger.error(
"Timeout in _remove_client_stop_internal client_id=%s, stack=%s",
client_id, s)
# Kill
g2.kill(block=True)
# Stat
Meters.aii("tcp.server.client_remove_timeout_internal")
except Exception as e:
logger.warning(
"Exception in _remove_client_stop_internal client_id=%s, ex=%s",
client_id, SolBase.extostr(e))
# Statistics
Meters.aii("tcp.server.client_connected", -1)
Meters.aii("tcp.server.client_remove_count")
# Log
logger.debug("client removed, id=%s, addr=%s",
old_client.get_client_id(),
old_client.get_client_addr())
return old_client
except Exception as e:
# Error
logger.warning("Exception, ex=%s", SolBase.extostr(e))
# Statistics
Meters.aii("tcp.server.client_remove_exception")
return None
finally:
if evt:
evt.set()
class AceClient(object):
def __init__(self, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = 0
# Did we get START command again? For seekback.
self._started_again = False
# Logger
logger = logging.getLogger('AceClient_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException(
"Socket creation error! Ace is not running? " + repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def _write(self, message):
try:
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0, seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
logger = logging.getLogger("AceClient_START")
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
errmsg = "START error!"
logger.error(errmsg)
raise AceException(errmsg)
except gevent.Timeout:
errmsg = "START timeout!"
logger.error(errmsg)
raise AceException(errmsg)
return result
def START(self, datatype, value):
'''
Start video method
'''
self._result = AsyncResult()
self._urlresult = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value))
contentinfo = self._getResult()
self._write(AceMessage.request.START(datatype.upper(), value))
self._getResult()
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
#logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin+4:self._request_key_begin+14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except:
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
_filename = urllib2.unquote(_contentinfo.get('files')[0][0])
self._result.set(_filename)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or (self._seekback and self._started_again):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seeback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
# logger.debug('Current position/last/buf: %s/%s/%s' % (self._position, self._position_last, self._position_buf))
if self._seekback and not self._started_again:
self._write(AceMessage.request.SEEK(str(int(self._position_last) - \
self._seekback)))
logger.debug('Seeking back')
self._started_again = True
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to {0} data {1}".format(self._status, repr(self._recvbuffer)))
if self._status == 'main:dl':
logger.debug("progress - speed {3}kb/s peers {6}".format(*self._recvbuffer.split(';')))
elif self._status == 'main:prebuf' or self._status == 'main:buf':
logger.debug("progress {1}% speed {5}kb/s peers {8}".format(*self._recvbuffer.split(';')))
elif self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
class TestRedisCache(unittest.TestCase):
"""
Test description
"""
currentTempDir = ""
def setUp(self):
"""
Setup
"""
# Reset counters
Meters.reset()
# Clear
self.redis_cache = None
# Key prefix
self.key_prefix = "rk_" + str(int(SolBase.mscurrent())) + "_"
# Temp redis : clear ALL
r = redis.Redis()
r.flushall()
del r
def tearDown(self):
"""
Stop
"""
if self.redis_cache:
logger.warning("Stopping redis_cache")
self.redis_cache.stop_cache()
self.redis_cache = None
# Temp redis : clear ALL
r = redis.Redis()
r.flushall()
del r
def test_start_stop(self):
"""
Test.
"""
# Alloc
self.redis_cache = RedisCache()
self.assertTrue(self.redis_cache._is_started)
# Stop
self.redis_cache.stop_cache()
self.assertFalse(self.redis_cache._is_started)
# Start
self.redis_cache.start_cache()
self.assertTrue(self.redis_cache._is_started)
# Stop
self.redis_cache.stop_cache()
self.assertFalse(self.redis_cache._is_started)
# Over
self.redis_cache = None
def test_init_no_stat(self):
"""
Test.
"""
# Alloc
m = RedisCache()
self.assertIsNotNone(m)
m.stop_cache()
def test_basic(self):
"""
Test.
"""
# Alloc
self.redis_cache = RedisCache()
# Get : must return nothing
o = self.redis_cache.get(self.key_prefix + "not_found")
self.assertIsNone(o)
# Put
self.redis_cache.put(self.key_prefix + "keyA", b"valA", 60000)
o = self.redis_cache.get(self.key_prefix + "keyA")
self.assertEqual(o, b"valA")
# Delete
self.redis_cache.remove(self.key_prefix + "keyA")
self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyA"))
# Non bytes injection : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.redis_cache.put(self.key_prefix + "toto", 12, 1000)
self.fail("Must fail")
except:
pass
# Non bytes injection : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.redis_cache.put(self.key_prefix + "toto", u"unicode_buffer",
1000)
self.fail("Must fail")
except:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.redis_cache.put(999, b"value", 60000)
self.fail("Put a key as non bytes,str MUST fail")
except Exception:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.redis_cache.remove(999)
self.fail("Remove a key as non bytes,str MUST fail")
except Exception:
pass
# Put/Remove
self.redis_cache.put(self.key_prefix + "todel", b"value", 60000)
self.assertEqual(self.redis_cache.get(self.key_prefix + "todel"),
b"value")
self.redis_cache.remove(self.key_prefix + "todel")
self.assertEqual(self.redis_cache.get(self.key_prefix + "todel"), None)
# Put
self.redis_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz",
60000)
self.assertEqual(
self.redis_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"),
b"zzz")
# Stop
self.redis_cache.stop_cache()
self.redis_cache = None
def test_basic_external_redis(self):
"""
Test.
"""
# External
r = Redis()
# Alloc
self.redis_cache = RedisCache(
pool_read_d=None,
pool_write_d=None,
redis_read_client=r,
redis_write_client=r,
)
# Get : must return nothing
o = self.redis_cache.get(self.key_prefix + "not_found")
self.assertIsNone(o)
# Put
self.redis_cache.put(self.key_prefix + "keyA", b"valA", 60000)
o = self.redis_cache.get(self.key_prefix + "keyA")
self.assertEqual(o, b"valA")
# Delete
self.redis_cache.remove(self.key_prefix + "keyA")
self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyA"))
# Non bytes injection : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.redis_cache.put(self.key_prefix + "toto", 12, 1000)
self.fail("Must fail")
except:
pass
# Non bytes injection : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.redis_cache.put(self.key_prefix + "toto", u"unicode_buffer",
1000)
self.fail("Must fail")
except:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.redis_cache.put(999, b"value", 60000)
self.fail("Put a key as non bytes,str MUST fail")
except Exception:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.redis_cache.remove(999)
self.fail("Remove a key as non bytes,str MUST fail")
except Exception:
pass
# Put/Remove
self.redis_cache.put(self.key_prefix + "todel", b"value", 60000)
self.assertEqual(self.redis_cache.get(self.key_prefix + "todel"),
b"value")
self.redis_cache.remove(self.key_prefix + "todel")
self.assertEqual(self.redis_cache.get(self.key_prefix + "todel"), None)
# Put
self.redis_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz",
60000)
self.assertEqual(
self.redis_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"),
b"zzz")
# Stop
self.redis_cache.stop_cache()
self.redis_cache = None
def test_basic_ttl(self):
"""
Test
:return:
"""
# Alloc
self.redis_cache = RedisCache()
# Put
self.redis_cache.put(self.key_prefix + "keyA", b"valA", 60000)
self.redis_cache.put(self.key_prefix + "keyB", b"valB", 1000)
logger.info("ms cur=%s", SolBase.mscurrent())
logger.info("A : %s", self.redis_cache.get(self.key_prefix + "keyA"))
logger.info("B : %s", self.redis_cache.get(self.key_prefix + "keyB"))
# Wait a bit
SolBase.sleep(2000)
logger.info("ms after sleep=%s", SolBase.mscurrent())
# A : must be present
# B : must be evicted (TTL elapsed)
self.assertEqual(self.redis_cache.get(self.key_prefix + "keyA"),
b"valA")
self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyB"))
self.assertEqual(Meters.aig("rcs.cache_put"), 2)
self.assertEqual(Meters.aig("rcs.cache_get_hit"), 3)
self.assertEqual(Meters.aig("rcs.cache_get_miss"), 1)
# Stop
self.redis_cache.stop_cache()
self.redis_cache = None
def test_max_item_size(self):
"""
Test
:return:
"""
# Alloc
self.redis_cache = RedisCache(max_single_item_bytes=2)
# Put
self.assertTrue(
self.redis_cache.put(self.key_prefix + "keyA", b"aa", 60000))
self.assertFalse(
self.redis_cache.put(self.key_prefix + "keyB", b"aaa", 60000))
logger.info("ms cur=%s", SolBase.mscurrent())
logger.info("A : %s", self.redis_cache.get(self.key_prefix + "keyA"))
logger.info("B : %s", self.redis_cache.get(self.key_prefix + "keyB"))
# A : must be present
# B : must be out of cache
self.assertEqual(self.redis_cache.get(self.key_prefix + "keyA"), b"aa")
self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyB"))
self.assertEqual(Meters.aig("rcs.cache_put"), 1)
self.assertEqual(Meters.aig("rcs.cache_get_hit"), 2)
self.assertEqual(Meters.aig("rcs.cache_get_miss"), 2)
# Stop
self.redis_cache.stop_cache()
self.redis_cache = None
# ========================
# BENCH
# ========================
def test_bench_greenlet_1_put_100_get_0_max_128000(self):
"""
Test
:return:
"""
self._go_greenlet(1, 100, 0, 128000)
def test_bench_greenlet_16_put_10_get_1_max_128000(self):
"""
Test
:return:
"""
self._go_greenlet(16, 1, 1, 128000)
def test_bench_greenlet_128_put_10_get_10_max_128000(self):
"""
Test
:return:
"""
self._go_greenlet(128, 10, 10, 128000)
def _go_greenlet(self, greenlet_count, put_count, get_count,
bench_item_count):
"""
Doc
:param greenlet_count: greenlet_count
:param put_count: put_count
:param get_count: get_count
:param bench_item_count: bench_item_count
"""
g_event = None
g_array = None
try:
# Settings
g_count = greenlet_count
g_ms = 10000
# Continue callback loop
self.callback_return = True
# Go
self.redis_cache = RedisCache()
# Item count
self.bench_item_count = bench_item_count
self.bench_put_weight = put_count
self.bench_get_weight = get_count
self.bench_ttl_min_ms = 1000
self.bench_ttl_max_ms = int(g_ms / 2)
# Go
self.run_event = Event()
self.exception_raised = 0
self.open_count = 0
self.thread_running = AtomicIntSafe()
self.thread_running_ok = AtomicIntSafe()
# Item per greenlet
item_per_greenlet = self.bench_item_count / g_count
# Signal
self.gorun_event = Event()
# Alloc greenlet
g_array = list()
g_event = list()
for _ in range(0, g_count):
greenlet = Greenlet()
g_array.append(greenlet)
g_event.append(Event())
# Run them
cur_idx = 0
for idx in range(0, len(g_array)):
greenlet = g_array[idx]
event = g_event[idx]
greenlet.spawn(self._run_cache_bench, event, cur_idx,
cur_idx + item_per_greenlet)
cur_idx += item_per_greenlet
SolBase.sleep(0)
# Signal
self.gorun_event.set()
# Wait a bit
dt = SolBase.mscurrent()
while SolBase.msdiff(dt) < g_ms:
SolBase.sleep(500)
# Stat
ms = SolBase.msdiff(dt)
sec = float(ms / 1000.0)
total_put = Meters.aig("rcs.cache_put")
per_sec_put = round(float(total_put) / sec, 2)
total_get = Meters.aig("rcs.cache_get_hit") + Meters.aig(
"rcs.cache_get_miss")
per_sec_get = round(float(total_get) / sec, 2)
logger.info(
"Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s",
self.open_count, self.thread_running.get(),
self.thread_running_ok.get(), per_sec_put, per_sec_get,
self.redis_cache)
self.assertEqual(self.exception_raised, 0)
# Over, signal
logger.info("Signaling, count=%s", self.open_count)
self.run_event.set()
# Wait
for g in g_event:
g.wait(30.0)
self.assertTrue(g.isSet())
g_event = None
g_array = None
# Log
Meters.write_to_logger()
finally:
self.run_event.set()
if g_event:
for g in g_event:
g.set()
if g_array:
for g in g_array:
g.kill()
if self.redis_cache:
self.redis_cache.stop_cache()
self.redis_cache = None
def _run_cache_bench(self, event, idx_min, idx_max):
"""
Run
:param idx_min: Index min
:param idx_max: Index max
"""
idx_max -= 1
# Wait
self.gorun_event.wait()
# Go
cur_count = 0
logger.debug("Entering now, idx_min=%s, idx_max=%s", idx_min, idx_max)
self.thread_running.increment()
self.thread_running_ok.increment()
try:
while not self.run_event.isSet():
cur_count += 1
try:
cur_item = random.randint(idx_min, idx_max)
s_cur_item = "%s" % cur_item
b_cur_item = SolBase.unicode_to_binary(s_cur_item, "utf-8")
cur_ttl = random.randint(self.bench_ttl_min_ms,
self.bench_ttl_max_ms)
for _ in range(0, self.bench_put_weight):
self.redis_cache.put(s_cur_item, b_cur_item, cur_ttl)
SolBase.sleep(0)
for _ in range(0, self.bench_get_weight):
v = self.redis_cache.get(s_cur_item)
if v:
self.assertEqual(v, b_cur_item)
SolBase.sleep(0)
except Exception as e:
self.exception_raised += 1
logger.warning("Ex=%s", SolBase.extostr(e))
self.thread_running_ok.increment(-1)
return
finally:
pass
finally:
self.assertGreater(cur_count, 0)
logger.debug("Exiting")
event.set()
self.thread_running.increment(-1)
class Config(object):
cmds = {
'heartbeat_interval': (int, None),
'command_port': (int, None),
'data_port': (int, None),
'pid_dir': (str, DEFAULT_PID_DIR),
'log_level': (str, DEFAULT_LOG_LEVEL),
'log_config': (str, None),
'antelope_orb_name': (str, None),
'antelope_orb_select': (str, None),
'antelope_orb_reject': (str, None),
'antelope_orb_after': (float, -1),
}
# TODO: What about command_port?
CONFIG_DEPS = set([
'heartbeat_interval',
'data_port',
'antelope_orb_name',
])
# When one of these changes, it signals the data server to restart
DATASERVER_DEPS = [
'data_port',
'antelope_orb_name',
'antelope_orb_select',
'antelope_orb_reject',
'antelope_orb_after',
]
def setval(self, name, val, *args, **kwargs):
setattr(self, name, val)
def __init__(self, options, cmdproc):
self.configuredevent = Event()
self.dataserverconfigupdate = Event()
self.heartbeatactive = Event()
for name, (converter, default) in self.cmds.iteritems():
self.cmdproc = cmdproc
# Initialize attr with default val
setattr(self, name, default)
# Create command to set attr
# cmdserver sends sock after val; will that mess this up?
setval = partial(self.setval, name)
cmdproc.setCmd(name, converter, setval)
# Set a default loglevel?
# update from config file
if hasattr(options, 'conffile') and options.conffile is not None:
self.readConfig(options.conffile)
# update from command line
if hasattr(options, 'verbose') and options.verbose is True:
self.log_level = 'debug'
if hasattr(options, 'command_port') and options.command_port is not None:
self.command_port = options.command_port
def readConfig(self, conffile):
with open(conffile, 'rU') as file:
for line in file:
self.cmdproc.processCmd(line)
def __setattr__(self, name, value):
super(Config, self).__setattr__(name, value)
if not self.configuredevent.isSet():
configured_attrs = set()
for attr in self.CONFIG_DEPS:
if hasattr(self, attr) and getattr(self, attr) is not None:
configured_attrs.add(attr)
if configured_attrs == self.CONFIG_DEPS:
self.configuredevent.set()
if name in self.DATASERVER_DEPS:
self.dataserverconfigupdate.set()
self.dataserverconfigupdate.clear()
@property
def heartbeat_interval(self):
return self._heartbeat_interval
@heartbeat_interval.setter
def heartbeat_interval(self, value):
self._heartbeat_interval = value
if value > 0:
self.heartbeatactive.set()
else:
self.heartbeatactive.clear()
@property
def log_level(self):
return self._log_level
@log_level.setter
def log_level(self, val):
levels = {
'error': logging.ERROR,
'warn': logging.WARNING,
'info': logging.INFO,
'debug': logging.DEBUG,
'mesg': logging.DEBUG,
}
try:
level = levels[val]
except KeyError:
log.error("Unknown logging level %s" % val)
else:
self._log_level = val
logging.getLogger().setLevel(level)
@property
def log_config(self):
return self._log_config
@log_config.setter
def log_config(self, val):
try:
ooi.logging.config.add_configuration(val)
self._log_config = val
except Exception:
log.error("Failed to read log config '%s'" % val, exc_info=True)
class AceClient(object):
def __init__(self, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = 0
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0;
# Logger
logger = logging.getLogger('AceClieimport tracebacknt_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException(
"Socket creation error! Ace is not running? " + repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger("AceClient_write")
logger.debug(message)
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0, seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException("Result not received")
except gevent.Timeout:
raise AceException("Timeout")
return result
def START(self, datatype, value):
'''
Start video method
'''
stream_type = 'output_format=http' if self._engine_version_code >= 3010500 and not AceConfig.vlcuse else ''
self._urlresult = AsyncResult()
self._write(AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state is not None and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, url):
self._result = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, {'url': url}))
return self._getResult()
def GETCID(self, datatype, url):
contentinfo = self.LOADASYNC(datatype, url)
self._cidresult = AsyncResult()
self._write(AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def GETCONTENTINFO(self, datatype, url):
contentinfo = self.LOADASYNC(datatype, url)
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter):
logger = logging.getLogger("StreamReader")
self._streamReaderState = 1
logger.debug("Opening video stream: %s" % url)
try:
connection = self._streamReaderConnection = urllib2.urlopen(url)
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return
if connection.getcode() != 200:
logger.error("Failed to open video stream %s" % connection)
return
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
while True:
data = None
clients = counter.getClients(cid)
try:
data = connection.read(AceConfig.readchunksize)
except:
break;
if data and clients:
with self._lock:
if len(self._streamReaderQueue) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" % str(c))
c.destroy()
elif not clients:
logger.debug("All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except urllib2.URLError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClients(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger("StreamReader")
c = self._streamReaderConnection
if c:
self._streamReaderConnection = None
c.close()
logger.debug("Video stream closed")
self._streamReaderQueue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
# logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'version_code=' in self._recvbuffer:
v = self._recvbuffer.find('version_code=')
self._engine_version_code = int(self._recvbuffer[v + 13:v + 20])
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except urllib2.URLError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
self._result.set(_contentinfo)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or self._started_again or not self._recvbuffer.endswith(' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seeback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS)
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
if self._seekback and not self._started_again:
self._write(AceMessage.request.SEEK(str(int(self._position_last) - \
self._seekback)))
logger.debug('Seeking back')
self._started_again = True
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._status == 'main:idle':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
elif self._recvbuffer.startswith('##') or len(self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
logger.debug("CID: %s" % self._recvbuffer)
def sendHeadersPT(self, client, code, headers):
client.handler.send_response(code)
for key, value in headers.items():
client.handler.send_header(key, value)
client.handler.end_headers()
def openStreamReaderPT(self, url, req_headers):
logger = logging.getLogger("openStreamReaderPT")
logger.debug("Opening video stream: %s" % url)
logger.debug("headers: %s" % req_headers)
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return None, None, None
request = urllib2.Request(url, headers=req_headers)
connection = self._streamReaderConnection = urllib2.urlopen(request, timeout=120)
code = connection.getcode()
if code not in (200, 206):
logger.error("Failed to open video stream %s" % connection)
return None, None, None
FORWARD_HEADERS = ['Content-Range',
'Connection',
'Keep-Alive',
'Content-Type',
'Accept-Ranges',
'X-Content-Duration',
'Content-Length',
]
SKIP_HEADERS = ['Server', 'Date']
response_headers = {}
for k in connection.info().headers:
if k.split(':')[0] not in (FORWARD_HEADERS + SKIP_HEADERS):
logger.debug('NEW HEADERS: %s' % k.split(':')[0])
for h in FORWARD_HEADERS:
if connection.info().getheader(h):
response_headers[h] = connection.info().getheader(h)
# self.connection.send_header(h, connection.info().getheader(h))
logger.debug('key=%s value=%s' % (h, connection.info().getheader(h)))
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
return connection, code, response_headers
def startStreamReaderPT(self, url, cid, counter, req_headers=None):
logger = logging.getLogger("StreamReaderPT")
self._streamReaderState = 1
# current_req_headers = req_headers
try:
while True:
data = None
clients = counter.getClientsPT(cid)
if not req_headers == self.req_headers:
self.req_headers = req_headers
connection, code, resp_headers = self.openStreamReaderPT(url, req_headers)
if not connection:
return
for c in clients:
try:
c.headers_sent
except:
self.sendHeadersPT(c, code, resp_headers)
c.headers_sent = True
# logger.debug("i")
try:
data = connection.read(AceConfig.readchunksize)
except:
break;
# logger.debug("d: %s c:%s" % (data, clients))
if data and clients:
with self._lock:
if len(self._streamReaderQueue) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" % str(c))
c.destroy()
elif not clients:
logger.debug("All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except urllib2.URLError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClientsPT(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
class AceClient(object):
def __init__(self, acehostslist, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = AceConfig.videoseekback
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0
# Logger
logger = logging.getLogger('AceClientimport tracebacknt_init')
# Try to connect AceStream engine
for AceEngine in acehostslist:
try:
self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1],
connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[
0], AceEngine[1], AceEngine[2]
logger.debug("Successfully connected to AceStream on %s:%d" %
(AceEngine[0], AceEngine[1]))
break
except:
logger.debug("The are no alive AceStream on %s:%d" %
(AceEngine[0], AceEngine[1]))
pass
# Spawning recvData greenlet
if self._socket:
gevent.spawn(self._recvData)
gevent.sleep()
else:
logger.error("The are no alive AceStream Engines found")
return
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
return # Already in the middle of destroying
logger = logging.getLogger("AceClient_destroy") # Logger
self._resumeevent.set(
) # We should resume video to prevent read greenlet deadlock
self._urlresult.set() # And to prevent getUrl deadlock
# Trying to disconnect
try:
logger.debug("Destroying AceStream client ...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
pass # Ignore exceptions on destroy
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger("AceClient_write")
logger.debug(message)
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_25_34,
product_key=AceConfig.acekey):
self._product_key = product_key
self._gender = gender
self._age = age
self._seekback = AceConfig.videoseekback
logger = logging.getLogger("AceClient_aceInit")
self._write(AceMessage.request.HELLO) # Sending HELLOBG
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?" # HELLOTS not resived from engine
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("AceInit ended")
def _getResult(self):
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException("Result not received from %s:%s" %
(AceConfig.acehost, AceConfig.aceAPIport))
except gevent.Timeout:
raise AceException("gevent_Timeout")
return result
def START(self, datatype, value, stream_type):
'''
Start video method
'''
if stream_type == 'hls':
stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \
+ ' transcode_mp3=' + str(AceConfig.transcode_mp3) \
+ ' transcode_ac3=' + str(AceConfig.transcode_ac3) \
+ ' preferred_audio_language=' + AceConfig.preferred_audio_language
else:
stream_type = 'output_format=http'
self._urlresult = AsyncResult()
self._write(
AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state is not None and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, params):
self._result = AsyncResult()
self._write(
AceMessage.request.LOADASYNC(
datatype.upper(),
random.randint(1, AceConfig.maxconns * 10000), params))
return self._getResult()
def CONTENTINFO(self, datatype, value):
dict = {
'torrent': 'url',
'infohash': 'infohash',
'raw': 'data',
'pid': 'content_id'
}
paramsdict = {
dict[datatype]: value,
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
return self.LOADASYNC(datatype, paramsdict)
def GETCID(self, datatype, url):
contentinfo = self.CONTENTINFO(datatype, url)
self._cidresult = AsyncResult()
self._write(
AceMessage.request.GETCID(contentinfo.get('checksum'),
contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def GETINFOHASH(self, datatype, url):
infohash = self.CONTENTINFO(datatype, url).get('infohash')
return '' if not infohash or infohash == '' else infohash
def getUrl(self, timeout=30):
logger = logging.getLogger("AceClient_getURL") # Logger
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "Engine response time exceeded. GetURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger("StreamReader")
logger.debug("Opening video stream: %s" % url)
self._streamReaderState = 1
transcoder = None
if 'range' in req_headers: del req_headers['range']
logger.debug("Get headers from client: %s" % req_headers)
try:
connection = self._streamReaderConnection = requests.get(
url, headers=req_headers, stream=True)
if connection.status_code not in (200, 206):
logger.error("Failed to open video stream %s" % url)
return None
if url.endswith('.m3u8'):
self._streamReaderConnection.headers = {
'Content-Type': 'application/octet-stream',
'Connection': 'Keep-Alive',
'Keep-Alive': 'timeout=15, max=100'
}
popen_params = {
"bufsize": AceConfig.readchunksize,
"stdout": PIPE,
"stderr": None,
"shell": False
}
if AceConfig.osplatform == 'Windows':
ffmpeg_cmd = 'ffmpeg.exe '
CREATE_NO_WINDOW = 0x08000000
CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
popen_params.update(creationflags=CREATE_NO_WINDOW
| DETACHED_PROCESS
| CREATE_NEW_PROCESS_GROUP)
else:
ffmpeg_cmd = 'ffmpeg '
ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url
transcoder = psutil.Popen(ffmpeg_cmd.split(), **popen_params)
out = transcoder.stdout
logger.warning(
"HLS stream detected. Ffmpeg transcoding started")
else:
out = connection.raw
except requests.exceptions.RequestException:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
else:
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
while True:
data = None
clients = counter.getClients(cid)
if clients:
try:
data = out.read(AceConfig.readchunksize)
except:
logger.debug("No data received")
pass
else:
with self._lock:
if len(self._streamReaderQueue
) == AceConfig.readcachesize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" %
c)
c.destroy()
else:
logger.debug(
"All clients disconnected - closing video stream")
break
finally:
self.closeStreamReader()
if transcoder:
try:
transcoder.kill()
logger.warning("Ffmpeg transcoding stoped")
except:
pass
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger("StreamReader")
c = self._streamReaderConnection
if c:
c.close()
self._streamReaderConnection = None
logger.debug("Video stream closed")
self._streamReaderQueue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def play(self):
self._write(AceMessage.request.PLAY)
def pause(self):
self._write(AceMessage.request.PAUSE)
def stop(self):
self._write(AceMessage.request.STOP)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n").strip()
logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
else:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'version_code=' in self._recvbuffer:
v = self._recvbuffer.find('version_code=')
self._engine_version_code = int(
self._recvbuffer[v + 13:v + 20])
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = self._recvbuffer[
self._request_key_begin +
4:self._request_key_begin + 14]
self._write(
AceMessage.request.READY_key(
self._request_key, self._product_key))
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
# NOTREADY
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
logger.error("Ace engine is not ready. Wrong auth?")
self._auth = None
self._authevent.set()
# LOADRESP
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
_contentinfo = json.loads(' '.join(
self._recvbuffer.split()[2:]))
if _contentinfo.get('status') == 100:
logger.error(
"LOADASYNC returned error with message: %s" %
_contentinfo.get('message'))
self._result.set(False)
else:
self._result.set(_contentinfo)
#logger.debug("Content info: %s", _contentinfo)
# START
elif self._recvbuffer.startswith(AceMessage.response.START):
if not self._seekback or self._started_again or not self._recvbuffer.endswith(
' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._urlresult.set(self._recvbuffer.split()[1])
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug("START received. Waiting for %s." %
AceMessage.response.LIVEPOS)
# STOP
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
# SHUTDOWN
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
# AUTH
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(
self._gender, self._age))
except:
pass
self._authevent.set()
# GETUSERDATA
elif self._recvbuffer.startswith(
AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
# LIVEPOS
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
if self._seekback and not self._started_again:
self._write(
AceMessage.request.LIVESEEK(
str(int(self._position_last) -
self._seekback)))
logger.debug('Seeking back')
self._started_again = True
# DOWNLOADSTOP
elif self._recvbuffer.startswith(
AceMessage.response.DOWNLOADSTOP):
self._state = self._recvbuffer.split()[1]
# STATE
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
# INFO
elif self._recvbuffer.startswith(AceMessage.response.INFO):
self._state = self._recvbuffer.split()[1]
# STATUS
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(
';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to %s" % self._status)
if self._status == 'main:err':
logger.error(self._status + ' with message ' +
self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' +
self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' +
self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._status == 'main:idle':
self._result.set(True)
# PAUSE
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
# RESUME
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
self._resumeevent.set()
# CID
elif self._recvbuffer.startswith('##') or len(
self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
logger.debug("CID: %s" % self._recvbuffer)
class AceClient:
def __init__(self, host, port, connect_timeout = 5, result_timeout = 5, debug = logging.ERROR):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Debug level
self._debug = debug
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Logger
logger = logging.getLogger('AceClient_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException("Socket creation error! Ace is not running? " + str(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def _write(self, message):
try:
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + str(e))
def aceInit(self, gender = AceConst.SEX_MALE, age = AceConst.AGE_18_24, product_key = None, pause_delay = 0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def START(self, datatype, value):
'''
Start video method
'''
# Logger
logger = logging.getLogger("AceClient_START")
self._result = AsyncResult()
self._urlresult = AsyncResult()
self._write(AceMessage.request.START(datatype.upper(), value))
try:
if not self._result.get(timeout = self._resulttimeout):
errmsg = "START error!"
logger.error(errmsg)
raise AceException(errmsg)
except gevent.Timeout:
errmsg = "START timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getUrl(self, timeout = 40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout = timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getPlayEvent(self, timeout = None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
self._resumeevent.wait(timeout = timeout)
return
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n", 1)
self._recvbuffer = self._recvbuffer.strip()
except:
# If something happened during read, abandon reader
# Should not ever happen
logger.error("Exception at socket read")
return
# Parsing everything
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'key=' in self._recvbuffer:
self._request_key = self._recvbuffer.split()[2].split('=')[1]
self._write(AceMessage.request.READY_key(self._request_key, self._product_key))
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
return
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to "+self._status)
if self._status == 'main:err':
logger.warning(self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
class AceClient(object):
def __init__(self, host, port, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Logger
logger = logging.getLogger('AceClient_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.info("Successfully connected with Ace!")
except Exception as e:
raise AceException(
"Socket creation error! Ace is not running? " + repr(e))
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def _write(self, message):
try:
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0):
self._product_key = product_key
self._gender = gender
self._age = age
# PAUSE/RESUME delay
self._pausedelay = pause_delay
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
logger = logging.getLogger("AceClient_START")
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
errmsg = "START error!"
logger.error(errmsg)
raise AceException(errmsg)
except gevent.Timeout:
errmsg = "START timeout!"
logger.error(errmsg)
raise AceException(errmsg)
return result
def START(self, datatype, value):
'''
Start video method
'''
self._result = AsyncResult()
self._urlresult = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value))
contentinfo = self._getResult()
self._write(AceMessage.request.START(datatype.upper(), value))
self._getResult()
return contentinfo
def getUrl(self, timeout=40):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "getURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
self._resumeevent.wait(timeout=timeout)
return
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'key=' in self._recvbuffer:
self._request_key = self._recvbuffer.split()[
2].split('=')[1]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key,
self._resulttimeout))
except urllib2.URLError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
_filename = urllib2.unquote(_contentinfo.get('files')[0][0])
self._result.set(_filename)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
class PriorityQueue(object):
"""A priority queue.
It is greenlet-safe, and offers the ability of changing priorities
and removing arbitrary items.
The queue is implemented as a custom min-heap. The priority is a
mix of a discrete priority level and of the timestamp. The
elements of the queue are QueueItems.
"""
PRIORITY_EXTRA_HIGH = 0
PRIORITY_HIGH = 1
PRIORITY_MEDIUM = 2
PRIORITY_LOW = 3
PRIORITY_EXTRA_LOW = 4
def __init__(self):
"""Create a priority queue."""
# The queue: a min-heap whose elements are of the form
# (priority, timestamp, item), where item is the actual data.
self._queue = []
# Reverse lookup for the items in the queue: a dictionary
# associating the index in the queue to each item.
self._reverse = {}
# Event to signal that there are items in the queue.
self._event = Event()
# Index of the next element that will be added to the queue.
self._next_index = 0
def _verify(self):
"""Make sure that the internal state of the queue is consistent.
This is used only for testing.
"""
if len(self._queue) != len(self._reverse):
return False
if len(self._queue) != self.length():
return False
if self.empty() != (self.length() == 0):
return False
if self._event.isSet() == self.empty():
return False
for item, idx in self._reverse.iteritems():
if self._queue[idx].item != item:
return False
return True
def __contains__(self, item):
"""Implement the 'in' operator for an item in the queue.
item (QueueItem): an item to search.
return (bool): True if item is in the queue.
"""
return item in self._reverse
def _swap(self, idx1, idx2):
"""Swap two elements in the queue, keeping their reverse
indices up to date.
idx1 (int): the index of the first element.
idx2 (int): the index of the second element.
"""
self._queue[idx1], self._queue[idx2] = \
self._queue[idx2], self._queue[idx1]
self._reverse[self._queue[idx1].item] = idx1
self._reverse[self._queue[idx2].item] = idx2
def _up_heap(self, idx):
"""Take the element in position idx up in the heap until its
position is the right one.
idx (int): the index of the element to lift.
return (int): the new index of the element.
"""
while idx > 0:
parent = (idx - 1) // 2
if self._queue[idx] < self._queue[parent]:
self._swap(parent, idx)
idx = parent
else:
break
return idx
def _down_heap(self, idx):
"""Take the element in position idx down in the heap until its
position is the right one.
idx (int): the index of the element to lower.
return (int): the new index of the element.
"""
last = len(self._queue) - 1
while 2 * idx + 1 <= last:
child = 2 * idx + 1
if 2 * idx + 2 <= last and \
self._queue[2 * idx + 2] < self._queue[child]:
child = 2 * idx + 2
if self._queue[child] < self._queue[idx]:
self._swap(child, idx)
idx = child
else:
break
return idx
def _updown_heap(self, idx):
"""Perform both operations of up_heap and down_heap on an
element.
idx (int): the index of the element to lift.
return (int): the new index of the element.
"""
idx = self._up_heap(idx)
return self._down_heap(idx)
def push(self, item, priority=None, timestamp=None):
"""Push an item in the queue. If timestamp is not specified,
uses the current time.
item (QueueItem): the item to add to the queue.
priority (int|None): the priority of the item, or None for
medium priority.
timestamp (datetime|None): the time of the submission, or None
to use now.
return (bool): false if the element was already in the queue
and was not pushed again, true otherwise..
"""
if item in self._reverse:
return False
if priority is None:
priority = PriorityQueue.PRIORITY_MEDIUM
if timestamp is None:
timestamp = make_datetime()
index = self._next_index
self._next_index += 1
self._queue.append(QueueEntry(item, priority, timestamp, index))
last = len(self._queue) - 1
self._reverse[item] = last
self._up_heap(last)
# Signal to listener greenlets that there might be something.
self._event.set()
return True
def top(self, wait=False):
"""Return the first element in the queue without extracting it.
wait (bool): if True, block until an element is present.
return (QueueEntry): first element in the queue.
raise (LookupError): on empty queue if wait was false.
"""
if not self.empty():
return self._queue[0]
else:
if not wait:
raise LookupError("Empty queue.")
else:
while True:
if self.empty():
self._event.wait()
continue
return self._queue[0]
def pop(self, wait=False):
"""Extract (and return) the first element in the queue.
wait (bool): if True, block until an element is present.
return (QueueEntry): first element in the queue.
raise (LookupError): on empty queue, if wait was false.
"""
top = self.top(wait)
last = len(self._queue) - 1
self._swap(0, last)
del self._reverse[top.item]
del self._queue[last]
# last is 0 when the queue becomes empty.
if last > 0:
self._down_heap(0)
else:
# Signal that there is nothing left for listeners.
self._event.clear()
return top
def remove(self, item):
"""Remove an item from the queue. Raise a KeyError if not present.
item (QueueItem): the item to remove.
return (QueueEntry): the complete entry removed.
raise (KeyError): if item not present.
"""
pos = self._reverse[item]
entry = self._queue[pos]
last = len(self._queue) - 1
self._swap(pos, last)
del self._reverse[item]
del self._queue[last]
if pos != last:
self._updown_heap(pos)
if self.empty():
self._event.clear()
return entry
def set_priority(self, item, priority):
"""Change the priority of an item inside the queue. Raises an
exception if the item is not in the queue.
item (QueueItem): the item whose priority needs to change.
priority (int): the new priority.
raise (LookupError): if item not present.
"""
pos = self._reverse[item]
self._queue[pos].priority = priority
self._updown_heap(pos)
def length(self):
"""Return the number of elements in the queue.
return (int): length of the queue
"""
return len(self._queue)
def empty(self):
"""Return if the queue is empty.
return (bool): is the queue empty?
"""
return self.length() == 0
def get_status(self):
"""Return the content of the queue. Note that the order may be not
correct, but the first element is the one at the top.
return ([QueueEntry]): a list of entries containing the
representation of the item, the priority and the
timestamp.
"""
return [{
'item': entry.item.to_dict(),
'priority': entry.priority,
'timestamp': make_timestamp(entry.timestamp)
} for entry in self._queue]
class AceClient:
def __init__(self, host, port, connect_timeout = 5, debug = logging.ERROR):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Debug level
self._debug = debug
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
self._resumeevent = Event()
# Logging init
logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s: %(message)s', datefmt='%d.%m.%Y %H:%M:%S', level=self._debug)
logger = logging.getLogger('AceClient_init')
try:
self._socket = telnetlib.Telnet(host, port, connect_timeout)
logger.debug("Successfully connected with Ace!")
except Exception as e:
raise AceException("Socket creation error! Ace is not running? " + str(e))
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
self.destroy()
def destroy(self):
logger = logging.getLogger("ace_destroy")
self._resumeevent.set()
self._urlresult.set()
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
if self._socket:
try:
logger.debug("Destroying client...")
self._write(AceMessage.request.SHUTDOWN)
self._shuttingDown.set()
except:
# Ignore exceptions on destroy
pass
def _write(self, message):
try:
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + str(e))
def aceInit(self, gender = AceConst.SEX_MALE, age = AceConst.GENDER_18_24, product_key = None, pause_delay = 0):
self._product_key = product_key
self._gender = gender
self._age = age
self._pausedelay = pause_delay
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(5):
logging.error("aceInit event timeout. Wrong key?")
return
if not self._auth:
logging.error("aceInit auth error. Wrong key?")
return
logging.debug("aceInit ended")
def START(self, datatype, value):
self._result = AsyncResult()
self._urlresult = AsyncResult()
if datatype.lower() == 'pid':
self._write(AceMessage.request.START('PID', {'content_id': value}))
elif datatype.lower() == 'torrent':
self._write(AceMessage.request.START('TORRENT', {'url': value}))
if not self._result.get():
raise AceException("START error!")
return
def getUrl(self):
res = self._urlresult.get()
if res:
return res
else:
return False
def getPlayEvent(self):
self._resumeevent.wait()
return
def _recvData(self):
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
if self._shuttingDown.isSet():
logger.debug("Shutting down is in the process, returning from _recvData...")
self._socket.close()
return
try:
self._recvbuffer = self._socket.read_until("\r\n", 1)
except Exception as e:
if self._shuttingDown.isSet():
logger.debug("Shutting down is in the process, returning from _recvData after socket_read...")
else:
raise e
# Parsing everything
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'key=' in self._recvbuffer:
self._request_key = self._recvbuffer.split()[2].split('=')[1]
self._write(AceMessage.request.READY_key(self._request_key, self._product_key))
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
# Not implemented yet
logger.error("Ace is not ready. Wrong auth?")
pass
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
self.destroy()
return
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to "+self._status)
if self._status == 'main:err':
logger.warning(self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set(False)
self._urlresult.set(False)
if self._status == 'main:starting':
self._result.set(True)
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep(self._pausedelay)
self._resumeevent.set()
