class _FormationCache(object): """A cache of instance data for a formation.""" def __init__(self, client, form_name, factory, interval): self.client = client self.form_name = form_name self.factory = factory self.interval = interval self._gthread = None self._cache = {} self._stopped = Event() self._running = Event() def start(self): self._gthread = gevent.spawn(self._loop) self._running.wait(timeout=0.1) return self def stop(self, timeout=None): self._stopped.set() self._gthread.join(timeout) def _update(self): self._cache = self.client.query_formation(self.form_name, self.factory) def _loop(self): while not self._stopped.isSet(): self._update() self._running.set() self._stopped.wait(self.interval) def query(self): """Return all instances and their names.""" return dict(self._cache)
class SyncBaseDeamon(object): def __init__(self, interval=5): self.ip_addr = socket.gethostbyname(socket.gethostname()) self.pid = os.getpid() self.channel = Queue(1) self.sync_interval = int(interval) self.exit_flag = Event() self.lock_path = "" self.logger = logging.getLogger() def exit(self): if not self.exit_flag.isSet(): self.exit_flag.set() self.channel.put('exit') else: self.logger.warn("Waiting exit signal been resolved") def terminate(self, code): zk.close() sys.exit(code) def run(self): greenlets = [] try: zk.create( self.lock_path, "%s:%s" % (self.ip_addr, self.pid), ephemeral=True, makepath=True ) except NodeExistsError: ret, _ = zk.get(self.lock_path) ip, pid = ret.split(':') self.logger.error("Servers syncing process is running on %s, pid: %s" % (ip, pid)) self.terminate(EXIT_CODES['leave_alone']) gevent.signal(signal.SIGTERM, self.exit) gevent.signal(signal.SIGINT, self.exit) gevent.signal(signal.SIGHUP, self.exit) greenlets.append(gevent.spawn(self.sync)) gevent.spawn(self.ticker) gevent.joinall(greenlets) self.logger.warn("syncing deamon exited") self.terminate(EXIT_CODES['normal_exit']) def ticker(self): while True: try: if self.channel.qsize() == 0: self.channel.put("sync", block=False) except Full: self.logger.error("something wrong occur in syncing thread") pass gevent.sleep(self.sync_interval) def sync(self, test=False): pass
class _Registration(object): """A service registration.""" def __init__(self, client, form_name, instance_name, data, interval=3): self.stopped = Event() self.client = client self.form_name = form_name self.instance_name = instance_name self.data = data self.interval = interval self.gthread = None def _loop(self): uri = '/%s/%s' % (self.form_name, self.instance_name) while not self.stopped.isSet(): response = self.client._request('PUT', uri, data=json.dumps(self.data)) self.stopped.wait(self.interval) def start(self): self.gthread = gevent.spawn(self._loop) return self def stop(self, timeout=None): self.stopped.set() self.gthread.join(timeout)
class Connection: def __init__( self, env, connections, handler, timeout ): self.ack_timeout = timeout self.ack_done = Event() self.ack_id = None self.connections = connections self.handler = handler self.server_event_queue = Queue() self.client_event_queue = Queue() self._id = uuid().hex self.env = deepcopydict( env ) self.env['connection'] = self spawn( self._kill_idle ) spawn\ ( self.handler , self.env , lambda: self.client_event_queue , self.server_event_queue.put ) def _check_next( self, confirm_ID ): result = self.server_event_queue.get() self.ack_id = confirm_ID self.current_result.set( (confirm_ID, result ) ) self._kill_idle() def next( self, confirm_ID, current_ID ): if confirm_ID == self.ack_id: self.ack_id = None self.ack_done.set() self.current_result = AsyncResult() spawn( self._check_next, current_ID ) result = self.current_result.get( ) return result def emit( self, event ): self.client_event_queue.put( event ) def _kill_idle( self ): self.ack_done.clear() self.ack_done.wait( timeout=self.ack_timeout ) if not self.ack_done.isSet(): self.close() def close( self ): self.client_event_queue.put( StopIteration ) self.server_event_queue.put( StopIteration ) del self.connections[ self._id ]
class Connection(object): def __init__(self, env, connections, handler, timeout): self.ack_timeout = timeout self.ack_done = Event() self.ack_id = None self.connections = connections self.handler = handler self.server_event_queue = Queue() self.client_event_queue = Queue() self._id = uuid().hex self.env = deepcopydict(env) self.env['connection'] = self spawn(self._kill_idle) spawn\ ( self.handler , self.env , lambda: self.client_event_queue , self.server_event_queue.put ) def _check_next(self, confirm_ID): result = self.server_event_queue.get() self.ack_id = confirm_ID self.current_result.set((confirm_ID, result)) self._kill_idle() def next(self, confirm_ID, current_ID): if confirm_ID == self.ack_id: self.ack_id = None self.ack_done.set() self.current_result = AsyncResult() spawn(self._check_next, current_ID) result = self.current_result.get() return result def emit(self, event): self.client_event_queue.put(event) def _kill_idle(self): self.ack_done.clear() self.ack_done.wait(timeout=self.ack_timeout) if not self.ack_done.isSet(): self.close() def close(self): self.client_event_queue.put(StopIteration) self.server_event_queue.put(StopIteration) del self.connections[self._id]
class ActionResult(object): def __init__(self, slave, action, state=PENDING): self.id_ = id(self) self.slave = slave self.action = action self._state = state self._text = "" self.event = Event() @property def state(self): return self._state @state.setter def state(self, state): if state not in (PENDING, RUNNING, SUCCESS, FAILURE): raise ValueError("Invalid state: %s" % state) self._state = state if state in (SUCCESS, FAILURE): self.event.set() @property def text(self): return self._text @text.setter def text(self, text): self._text = text def is_done(self): if self.event.isSet(): return True else: return False def serialize(self, include_requestid=False): data = {"state": self.state, "text": self.text} if include_requestid: data["requestid"] = self.id_ return data def wait(self, timeout=None): return self.event.wait(timeout)
class ActionResult(object): """Contains basic information about the result of a specific Action.""" def __init__(self, slave, action, state=PENDING, request_timestamp=0, start_timestamp=0, finish_timestamp=0): self.id_ = id(self) self.slave = slave self.action = action self._state = state self._text = "" if not request_timestamp: request_timestamp = time.time() self._request_timestamp = request_timestamp self._start_timestamp = start_timestamp self._finish_timestamp = finish_timestamp self.event = Event() @property def state(self): return self._state @state.setter def state(self, state): if state not in (PENDING, RUNNING, SUCCESS, FAILURE): raise ValueError("Invalid state: %s" % state) self._state = state if state in (SUCCESS, FAILURE): self.event.set() @property def text(self): return self._text @text.setter def text(self, text): self._text = text @property def request_timestamp(self): return self._request_timestamp @request_timestamp.setter def request_timestamp(self, timestamp): self._request_timestamp = timestamp @property def start_timestamp(self): return self._start_timestamp @start_timestamp.setter def start_timestamp(self, timestamp): self._start_timestamp = timestamp @property def finish_timestamp(self): return self._finish_timestamp @finish_timestamp.setter def finish_timestamp(self, timestamp): self._finish_timestamp = timestamp def is_done(self): if self.event.isSet(): return True else: return False def to_dict(self, include_requestid=False): """Returns the state and text of this ActionResult in a dict. If include_requestid is True, "requestid" will also be present. Example: .. code-block:: python { "state": 2, "text": "Great success!", "request_timestamp": 1392414314, "start_timestamp": 1392414315, "finish_timestamp": 1392414316, "requestid": "234567832" } """ data = {"state": self.state, "text": self.text, "request_timestamp": self._request_timestamp, "start_timestamp": self._start_timestamp, "finish_timestamp": self._finish_timestamp} if include_requestid: data["requestid"] = self.id_ return data def wait(self, timeout=None): return self.event.wait(timeout)
class AceClient(PVRClient): ENGINE_TYPE = ('pid', 'torrent') def __init__(self, vlc_client, host, port, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Logger logger = logging.getLogger('AceClient_init') self.vlc_client = vlc_client try: self._socket = telnetlib.Telnet(host, port, connect_timeout) logger.info("Successfully connected with Ace!") except Exception as e: raise AceException( "Socket creation error! Ace is not running? " + repr(e)) # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self.vlc_client.stopBroadcast(self.stream_name) self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def _write(self, message): try: self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def getType(self): return AceClient.ENGINE_TYPE def init(self, stream_name, gender=PVRConst.SEX_MALE, age=PVRConst.AGE_18_24, product_key=None, pause_delay=0): self._product_key = product_key self._gender = gender self._age = age self.stream_name = stream_name # PAUSE/RESUME delay self._pausedelay = pause_delay # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def _getResult(self): # Logger logger = logging.getLogger("AceClient_START") try: result = self._result.get(timeout=self._resulttimeout) if not result: errmsg = "START error!" logger.error(errmsg) raise AceException(errmsg) except gevent.Timeout: errmsg = "START timeout!" logger.error(errmsg) raise AceException(errmsg) return result def START(self, datatype, value): ''' Start video method ''' self._result = AsyncResult() self._urlresult = AsyncResult() self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value)) contentinfo = self._getResult() self._write(AceMessage.request.START(datatype.upper(), value)) self._getResult() return contentinfo def getUrl(self, timeout=40): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "getURL timeout!" logger.error(errmsg) raise AceException(errmsg) def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' self._resumeevent.wait(timeout=timeout) return def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n") self._recvbuffer = self._recvbuffer.strip() except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return if self._recvbuffer: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'key=' in self._recvbuffer: self._request_key_begin = self._recvbuffer.find('key=') self._request_key = \ self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14] try: self._write(AceMessage.request.READY_key( self._request_key, self._product_key)) except urllib2.URLError as e: logger.error("Can't connect to keygen server! " + \ repr(e)) self._auth = False self._authevent.set() self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") self._auth = False self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): # LOADRESP _contentinfo_raw = self._recvbuffer.split()[2:] _contentinfo_raw = ' '.join(_contentinfo_raw) _contentinfo = json.loads(_contentinfo_raw) if _contentinfo.get('status') == 100: logger.error("LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: logger.debug("Content info: %s", _contentinfo) _filename = urllib2.unquote(_contentinfo.get('files')[0][0]) self._result.set(_filename) elif self._recvbuffer.startswith(AceMessage.response.START): # START try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None elif self._recvbuffer.startswith(AceMessage.response.STOP): pass elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to " + self._status) if self._status == 'main:err': logger.error( self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep(self._pausedelay) self._resumeevent.set()
class PriorityQueue(object): """A priority queue. It is greenlet-safe, and offers the ability of changing priorities and removing arbitrary items. The queue is implemented as a custom min-heap. The priority is a mix of a discrete priority level and of the timestamp. The elements of the queue are QueueItems. """ PRIORITY_EXTRA_HIGH = 0 PRIORITY_HIGH = 1 PRIORITY_MEDIUM = 2 PRIORITY_LOW = 3 PRIORITY_EXTRA_LOW = 4 def __init__(self): """Create a priority queue.""" # The queue: a min-heap whose elements are of the form # (priority, timestamp, item), where item is the actual data. self._queue = [] # Reverse lookup for the items in the queue: a dictionary # associating the index in the queue to each item. self._reverse = {} # Event to signal that there are items in the queue. self._event = Event() # Index of the next element that will be added to the queue. self._next_index = 0 def __len__(self): return len(self._queue) def _verify(self): """Make sure that the internal state of the queue is consistent. This is used only for testing. """ if len(self._queue) != len(self._reverse): return False if len(self._queue) != self.length(): return False if self.empty() != (self.length() == 0): return False if self._event.isSet() == self.empty(): return False for item, idx in self._reverse.iteritems(): if self._queue[idx].item != item: return False return True def __contains__(self, item): """Implement the 'in' operator for an item in the queue. item (QueueItem): an item to search. return (bool): True if item is in the queue. """ return item in self._reverse def _swap(self, idx1, idx2): """Swap two elements in the queue, keeping their reverse indices up to date. idx1 (int): the index of the first element. idx2 (int): the index of the second element. """ self._queue[idx1], self._queue[idx2] = \ self._queue[idx2], self._queue[idx1] self._reverse[self._queue[idx1].item] = idx1 self._reverse[self._queue[idx2].item] = idx2 def _up_heap(self, idx): """Take the element in position idx up in the heap until its position is the right one. idx (int): the index of the element to lift. return (int): the new index of the element. """ while idx > 0: parent = (idx - 1) // 2 if self._queue[idx] < self._queue[parent]: self._swap(parent, idx) idx = parent else: break return idx def _down_heap(self, idx): """Take the element in position idx down in the heap until its position is the right one. idx (int): the index of the element to lower. return (int): the new index of the element. """ last = len(self._queue) - 1 while 2 * idx + 1 <= last: child = 2 * idx + 1 if 2 * idx + 2 <= last and \ self._queue[2 * idx + 2] < self._queue[child]: child = 2 * idx + 2 if self._queue[child] < self._queue[idx]: self._swap(child, idx) idx = child else: break return idx def _updown_heap(self, idx): """Perform both operations of up_heap and down_heap on an element. idx (int): the index of the element to lift. return (int): the new index of the element. """ idx = self._up_heap(idx) return self._down_heap(idx) def push(self, item, priority=None, timestamp=None): """Push an item in the queue. If timestamp is not specified, uses the current time. item (QueueItem): the item to add to the queue. priority (int|None): the priority of the item, or None for medium priority. timestamp (datetime|None): the time of the submission, or None to use now. return (bool): false if the element was already in the queue and was not pushed again, true otherwise.. """ if item in self._reverse: return False if priority is None: priority = PriorityQueue.PRIORITY_MEDIUM if timestamp is None: timestamp = make_datetime() index = self._next_index self._next_index += 1 self._queue.append(QueueEntry(item, priority, timestamp, index)) last = len(self._queue) - 1 self._reverse[item] = last self._up_heap(last) # Signal to listener greenlets that there might be something. self._event.set() return True def top(self, wait=False): """Return the first element in the queue without extracting it. wait (bool): if True, block until an element is present. return (QueueEntry): first element in the queue. raise (LookupError): on empty queue if wait was false. """ if not self.empty(): return self._queue[0] else: if not wait: raise LookupError("Empty queue.") else: while True: if self.empty(): self._event.wait() continue return self._queue[0] def pop(self, wait=False): """Extract (and return) the first element in the queue. wait (bool): if True, block until an element is present. return (QueueEntry): first element in the queue. raise (LookupError): on empty queue, if wait was false. """ top = self.top(wait) last = len(self._queue) - 1 self._swap(0, last) del self._reverse[top.item] del self._queue[last] # last is 0 when the queue becomes empty. if last > 0: self._down_heap(0) else: # Signal that there is nothing left for listeners. self._event.clear() return top def remove(self, item): """Remove an item from the queue. Raise a KeyError if not present. item (QueueItem): the item to remove. return (QueueEntry): the complete entry removed. raise (KeyError): if item not present. """ pos = self._reverse[item] entry = self._queue[pos] last = len(self._queue) - 1 self._swap(pos, last) del self._reverse[item] del self._queue[last] if pos != last: self._updown_heap(pos) if self.empty(): self._event.clear() return entry def set_priority(self, item, priority): """Change the priority of an item inside the queue. Raises an exception if the item is not in the queue. item (QueueItem): the item whose priority needs to change. priority (int): the new priority. raise (LookupError): if item not present. """ pos = self._reverse[item] self._queue[pos].priority = priority self._updown_heap(pos) def length(self): """Return the number of elements in the queue. return (int): length of the queue """ return len(self._queue) def empty(self): """Return if the queue is empty. return (bool): is the queue empty? """ return self.length() == 0 def get_status(self): """Return the content of the queue. Note that the order may be not correct, but the first element is the one at the top. return ([QueueEntry]): a list of entries containing the representation of the item, the priority and the timestamp. """ return [{'item': entry.item.to_dict(), 'priority': entry.priority, 'timestamp': make_timestamp(entry.timestamp)} for entry in self._queue]
class AceClient(object): def __init__(self, acehostslist, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current video position self._position = None # Available video position (loaded data) self._position_last = None # Buffered video pieces self._position_buf = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Result for GETCID() self._cidresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Seekback seconds. self._seekback = AceConfig.videoseekback # Did we get START command again? For seekback. self._started_again = False self._idleSince = time.time() self._streamReaderConnection = None self._streamReaderState = None self._lock = threading.Condition(threading.Lock()) self._streamReaderQueue = gevent.queue.Queue( maxsize=AceConfig.readcachesize) # Ring buffer self._engine_version_code = 0 # Logger logger = logging.getLogger('AceClientimport tracebacknt_init') # Try to connect AceStream engine for AceEngine in acehostslist: try: self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout) AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[ 0], AceEngine[1], AceEngine[2] logger.debug('Successfully connected to AceStream on %s:%d' % (AceEngine[0], AceEngine[1])) break except: logger.debug('The are no alive AceStream on %s:%d' % (AceEngine[0], AceEngine[1])) pass # Spawning recvData greenlet if self._socket: gevent.spawn(self._recvData) gevent.sleep() else: logger.error('The are no alive AceStream Engines found') return def destroy(self): ''' AceClient Destructor ''' logger = logging.getLogger('AceClient_destroy') # Logger if self._shuttingDown.isSet(): return # Already in the middle of destroying self._resumeevent.set( ) # We should resume video to prevent read greenlet deadlock self._urlresult.set() # And to prevent getUrl deadlock # Trying to disconnect try: logger.debug('Destroying AceStream client.....') self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: pass # Ignore exceptions on destroy finally: self._shuttingDown.set() def reset(self): self._started_again = False self._idleSince = time.time() self._streamReaderState = None def _write(self, message): try: logger = logging.getLogger('AceClient_write') logger.debug('>>> %s' % message) self._socket.write('%s\r\n' % message) except EOFError as e: raise AceException('Write error! %s' % repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_25_34, product_key=AceConfig.acekey): self._product_key = product_key self._gender = gender self._age = age self._seekback = AceConfig.videoseekback self._started_again = False logger = logging.getLogger('AceClient_aceInit') self._write(AceMessage.request.HELLO) # Sending HELLOBG if not self._authevent.wait(self._resulttimeout): errmsg = 'Authentication timeout during AceEngine init. Wrong key?' # HELLOTS not resived from engine logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = 'Authentication error during AceEngine init. Wrong key?' logger.error(errmsg) raise AceException(errmsg) return # Display download_stopped massage if self._engine_version_code >= 3003600: self._write(AceMessage.request.SETOPTIONS) def _getResult(self): try: result = self._result.get(timeout=self._resulttimeout) if not result: raise AceException('Result not received from %s:%s' % (AceConfig.acehost, AceConfig.aceAPIport)) except gevent.Timeout: raise AceException('gevent_Timeout') return result def START(self, datatype, value, stream_type): ''' Start video method ''' if stream_type == 'hls' and self._engine_version_code >= 3010500: stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \ + ' transcode_mp3=' + str(AceConfig.transcode_mp3) \ + ' transcode_ac3=' + str(AceConfig.transcode_ac3) \ + ' preferred_audio_language=' + AceConfig.preferred_audio_language else: stream_type = 'output_format=http' self._urlresult = AsyncResult() self._write( AceMessage.request.START(datatype.upper(), value, stream_type)) self._getResult() def STOP(self): ''' Stop video method ''' if self._state and self._state != '0': self._result = AsyncResult() self._write(AceMessage.request.STOP) self._getResult() def LOADASYNC(self, datatype, params): self._result = AsyncResult() self._write( AceMessage.request.LOADASYNC( datatype.upper(), random.randint(1, AceConfig.maxconns * 10000), params)) return self._getResult() def GETCONTENTINFO(self, datatype, value): paramsdict = { datatype: value, 'developer_id': '0', 'affiliate_id': '0', 'zone_id': '0' } return self.LOADASYNC(datatype, paramsdict) def GETCID(self, datatype, url): contentinfo = self.GETCONTENTINFO(datatype, url) self._cidresult = AsyncResult() self._write( AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0)) cid = self._cidresult.get(True, 5) return '' if not cid or cid == '' else cid[2:] def getUrl(self, timeout=30): logger = logging.getLogger('AceClient_getURL') # Logger try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = 'Engine response time exceeded. GetURL timeout!' logger.error(errmsg) raise AceException(errmsg) def startStreamReader(self, url, cid, counter, req_headers=None): logger = logging.getLogger('StreamReader') logger.debug('Open video stream: %s' % url) self._streamReaderState = 1 transcoder = None if 'range' in req_headers: del req_headers['range'] logger.debug('Get headers from client: %s' % req_headers) with requests.get(url, headers=req_headers, stream=True, timeout=(5, None)) as self._streamReaderConnection: try: if self._streamReaderConnection.status_code not in (200, 206): logger.error('Failed to open video stream %s' % url) return None if url.endswith('.m3u8'): self._streamReaderConnection.headers = { 'Content-Type': 'application/octet-stream', 'Connection': 'Keep-Alive', 'Keep-Alive': 'timeout=15, max=100' } popen_params = { "bufsize": AceConfig.readchunksize, "stdout": PIPE, "stderr": None, "shell": False } if AceConfig.osplatform == 'Windows': ffmpeg_cmd = 'ffmpeg.exe ' CREATE_NO_WINDOW = 0x08000000 CREATE_NEW_PROCESS_GROUP = 0x00000200 DETACHED_PROCESS = 0x00000008 popen_params.update(creationflags=CREATE_NO_WINDOW | DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP) else: ffmpeg_cmd = 'ffmpeg ' ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url transcoder = Popen(ffmpeg_cmd.split(), **popen_params) out = transcoder.stdout logger.warning( 'HLS stream detected. Ffmpeg transcoding started') else: out = self._streamReaderConnection.raw except requests.exceptions.RequestException: logger.error('Failed to open video stream %s' % url) logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) else: with self._lock: self._streamReaderState = 2 self._lock.notifyAll() self.play_event() while 1: self.getPlayEvent( ) # Wait for PlayEvent (stop/resume sending data from AceEngine to streamReaderQueue) clients = counter.getClients(cid) try: data = out.read(AceConfig.readchunksize) except: data = None if data is not None and clients: if self._streamReaderQueue.full(): self._streamReaderQueue.get() self._streamReaderQueue.put(data) for c in clients: try: c.queue.put(data, timeout=5) except gevent.queue.Full: #Queue.Full client does not read data from buffer until 5sec - disconnect it if len(clients) > 1: logger.debug('Disconnecting client: %s' % c.handler.clientip) c.destroy() elif counter.count(cid) == 0: logger.debug( 'All clients disconnected - broadcast stoped') break else: logger.warning('No data received - broadcast stoped') counter.deleteAll(cid) break finally: with self._lock: self._streamReaderState = None self._lock.notifyAll() if transcoder: try: transcoder.kill() logger.warning('Ffmpeg transcoding stoped') except: pass def closeStreamReader(self): logger = logging.getLogger('StreamReader') c = self._streamReaderConnection if c: logger.debug('Close video stream: %s' % c.url) c.close() self._streamReaderConnection = None self._streamReaderQueue.queue.clear() def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' return self._resumeevent.wait(timeout=timeout) # EVENTS from client to AceEngine def play_event(self): self._write(AceMessage.request.PLAYEVENT) def pause_event(self): self._write(AceMessage.request.PAUSEEVENT) def stop_event(self): self._write(AceMessage.request.STOPEVENT) # END EVENTS def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while 1: gevent.sleep() try: self._recvbuffer = self._socket.read_until('\r\n').strip() logger.debug( '<<< %s' % requests.compat.unquote(self._recvbuffer).decode('utf8')) except: # If something happened during read, abandon reader. logger.error('Exception at socket read. AceClient destroyed') if not self._shuttingDown.isSet(): self._shuttingDown.set() return else: # Parsing everything only if the string is not empty # HELLOTS if self._recvbuffer.startswith(AceMessage.response.HELLO): try: params = { k: v for k, v in (x.split('=') for x in self._recvbuffer.split() if '=' in x) } except: logger.error("Can't parse HELLOTS") params = {} if 'version_code' in params: self._engine_version_code = params['version_code'] if 'key' in params: self._write( AceMessage.request.READY_key( params['key'], self._product_key)) self._request_key = None else: self._write(AceMessage.request.READY_nokey) # NOTREADY elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): self._auth = None self._authevent.set() logger.error('AceEngine is not ready. Wrong auth?') # AUTH elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] self._write( AceMessage.request.USERDATA( self._gender, self._age)) except: pass self._authevent.set() # GETUSERDATA elif self._recvbuffer.startswith( AceMessage.response.GETUSERDATA): raise AceException('You should init me first!') # LOADRESP elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): _contentinfo = json.loads( requests.compat.unquote(' '.join( self._recvbuffer.split()[2:])).decode('utf8')) if _contentinfo.get('status') == 100: logger.error( 'LOADASYNC returned error with message: %s' % _contentinfo.get('message')) self._result.set(False) else: self._result.set(_contentinfo) # START elif self._recvbuffer.startswith(AceMessage.response.START): if not self._seekback or self._started_again or not self._recvbuffer.endswith( ' stream=1'): # If seekback is disabled, we use link in first START command. # If seekback is enabled, we wait for first START command and # ignore it, then do seekback in first EVENT position command # AceStream sends us STOP and START again with new link. # We use only second link then. try: self._urlresult.set(self._recvbuffer.split()[1]) self._resumeevent.set() except IndexError as e: self._url = None else: logger.debug('START received. Waiting for %s.' % AceMessage.response.LIVEPOS) # STATE elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] if self._state in ('0', '1'): self._result.set(True) #idle, starting elif self._state == '6': self._result.set(False) # error # STATUS elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._status = self._recvbuffer.split()[1].split(';') if 'main:err' in set( self._status): # main:err;error_id;error_message logger.error('%s with message %s' % (self._status[0], self._status[2])) self._result.set_exception( AceException('%s with message %s' % (self._status[0], self._status[2]))) self._urlresult.set_exception( AceException('%s with message %s' % (self._status[0], self._status[2]))) # LIVEPOS elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS): if self._seekback and not self._started_again: try: params = { k: v for k, v in (x.split('=') for x in self._recvbuffer.split() if '=' in x) } self._write( AceMessage.request.LIVESEEK( int(params['last']) - self._seekback)) logger.debug('Seeking back.....') self._started_again = True except: logger.error("Can't parse %s" % AceMessage.response.LIVEPOS) # CID elif self._recvbuffer.startswith('##') or len( self._recvbuffer) == 0: self._cidresult.set(self._recvbuffer) #DOWNLOADSTOP elif self._recvbuffer.startswith( AceMessage.response.DOWNLOADSTOP): pass # INFO elif self._recvbuffer.startswith(AceMessage.response.INFO): pass # SHOWURL elif self._recvbuffer.startswith(AceMessage.response.SHOWURL): pass # CANSAVE elif self._recvbuffer.startswith(AceMessage.response.CANSAVE): pass # PAUSE elif self._recvbuffer.startswith(AceMessage.response.PAUSE): self.pause_event() self._resumeevent.clear() # RESUME elif self._recvbuffer.startswith(AceMessage.response.RESUME): self.play_event() self._resumeevent.set() # STOP elif self._recvbuffer.startswith(AceMessage.response.STOP): pass # SHUTDOWN elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): self._socket.get_socket().shutdown(SHUT_WR) self._recvbuffer = self._socket.read_all() self._socket.close() logger.debug('AceClient destroyed') return
class AceClient(object): def __init__(self, host, port, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current video position self._position = None # Available video position (loaded data) self._position_last = None # Buffered video pieces self._position_buf = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Result for GETCID() self._cidresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Seekback seconds. self._seekback = 0 # Did we get START command again? For seekback. self._started_again = False self._idleSince = time.time() self._lock = threading.Condition(threading.Lock()) self._streamReaderConnection = None self._streamReaderState = None self._streamReaderQueue = deque() self._engine_version_code = 0 # Logger logger = logging.getLogger('AceClieimport tracebacknt_init') try: self._socket = telnetlib.Telnet(host, port, connect_timeout) logger.info("Successfully connected with Ace!") except Exception as e: raise AceException("Socket creation error! Ace is not running? " + repr(e)) # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def reset(self): self._started_again = False self._idleSince = time.time() self._streamReaderState = None def _write(self, message): try: logger = logging.getLogger("AceClient_write") logger.debug(message) self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0, seekback=0): self._product_key = product_key self._gender = gender self._age = age # PAUSE/RESUME delay self._pausedelay = pause_delay # Seekback seconds self._seekback = seekback # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def _getResult(self): # Logger try: result = self._result.get(timeout=self._resulttimeout) if not result: raise AceException("Result not received") except gevent.Timeout: raise AceException("Timeout") return result def START(self, datatype, value): ''' Start video method ''' stream_type = 'output_format=http' if self._engine_version_code >= 3010500 and not AceConfig.vlcuse else '' self._urlresult = AsyncResult() self._write( AceMessage.request.START(datatype.upper(), value, stream_type)) self._getResult() def STOP(self): ''' Stop video method ''' if self._state is not None and self._state != '0': self._result = AsyncResult() self._write(AceMessage.request.STOP) self._getResult() def LOADASYNC(self, datatype, url): self._result = AsyncResult() self._write( AceMessage.request.LOADASYNC(datatype.upper(), 0, {'url': url})) return self._getResult() def GETCID(self, datatype, url): contentinfo = self.LOADASYNC(datatype, url) self._cidresult = AsyncResult() self._write( AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0)) cid = self._cidresult.get(True, 5) return '' if not cid or cid == '' else cid[2:] def GETCONTENTINFO(self, datatype, url): contentinfo = self.LOADASYNC(datatype, url) return contentinfo def getUrl(self, timeout=40): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "getURL timeout!" logger.error(errmsg) raise AceException(errmsg) def startStreamReader(self, url, cid, counter): logger = logging.getLogger("StreamReader") self._streamReaderState = 1 logger.debug("Opening video stream: %s" % url) try: connection = self._streamReaderConnection = urllib2.urlopen(url) if url.endswith('.m3u8'): logger.debug("Can't stream HLS in non VLC mode: %s" % url) return if connection.getcode() != 200: logger.error("Failed to open video stream %s" % connection) return with self._lock: self._streamReaderState = 2 self._lock.notifyAll() while True: data = None clients = counter.getClients(cid) try: data = connection.read(AceConfig.readchunksize) except: break if data and clients: with self._lock: if len(self._streamReaderQueue ) == AceConfig.readcachesize: self._streamReaderQueue.popleft() self._streamReaderQueue.append(data) for c in clients: try: c.addChunk(data, 5.0) except Queue.Full: if len(clients) > 1: logger.debug("Disconnecting client: %s" % str(c)) c.destroy() elif not clients: logger.debug( "All clients disconnected - closing video stream") break else: logger.warning("No data received") break except urllib2.URLError: logger.error("Failed to open video stream") logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) if counter.getClients(cid): logger.error("Failed to read video stream") finally: self.closeStreamReader() with self._lock: self._streamReaderState = 3 self._lock.notifyAll() counter.deleteAll(cid) def closeStreamReader(self): logger = logging.getLogger("StreamReader") c = self._streamReaderConnection if c: self._streamReaderConnection = None c.close() logger.debug("Video stream closed") self._streamReaderQueue.clear() def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' return self._resumeevent.wait(timeout=timeout) def pause(self): self._write(AceMessage.request.PAUSE) def play(self): self._write(AceMessage.request.PLAY) def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n") self._recvbuffer = self._recvbuffer.strip() # logger.debug('<<< ' + self._recvbuffer) except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return if self._recvbuffer: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'version_code=' in self._recvbuffer: v = self._recvbuffer.find('version_code=') self._engine_version_code = int( self._recvbuffer[v + 13:v + 20]) if 'key=' in self._recvbuffer: self._request_key_begin = self._recvbuffer.find('key=') self._request_key = \ self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14] try: self._write( AceMessage.request.READY_key( self._request_key, self._product_key)) except urllib2.URLError as e: logger.error("Can't connect to keygen server! " + \ repr(e)) self._auth = False self._authevent.set() self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") self._auth = False self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): # LOADRESP _contentinfo_raw = self._recvbuffer.split()[2:] _contentinfo_raw = ' '.join(_contentinfo_raw) _contentinfo = json.loads(_contentinfo_raw) if _contentinfo.get('status') == 100: logger.error( "LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: logger.debug("Content info: %s", _contentinfo) self._result.set(_contentinfo) elif self._recvbuffer.startswith(AceMessage.response.START): # START if not self._seekback or self._started_again or not self._recvbuffer.endswith( ' stream=1'): # If seekback is disabled, we use link in first START command. # If seekback is enabled, we wait for first START command and # ignore it, then do seeback in first EVENT position command # AceStream sends us STOP and START again with new link. # We use only second link then. try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None else: logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS) elif self._recvbuffer.startswith(AceMessage.response.STOP): pass elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA( self._gender, self._age)) except: pass self._authevent.set() elif self._recvbuffer.startswith( AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS): self._position = self._recvbuffer.split() self._position_last = self._position[2].split('=')[1] self._position_buf = self._position[9].split('=')[1] self._position = self._position[4].split('=')[1] if self._seekback and not self._started_again: self._write(AceMessage.request.SEEK(str(int(self._position_last) - \ self._seekback))) logger.debug('Seeking back') self._started_again = True elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split( ';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to " + self._status) if self._status == 'main:err': logger.error(self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._status == 'main:idle': self._result.set(True) elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep(self._pausedelay) self._resumeevent.set() elif self._recvbuffer.startswith('##') or len( self._recvbuffer) == 0: self._cidresult.set(self._recvbuffer) logger.debug("CID: %s" % self._recvbuffer) def sendHeadersPT(self, client, code, headers): client.handler.send_response(code) for key, value in headers.items(): client.handler.send_header(key, value) client.handler.end_headers() def openStreamReaderPT(self, url, req_headers): logger = logging.getLogger("openStreamReaderPT") logger.debug("Opening video stream: %s" % url) logger.debug("headers: %s" % req_headers) if url.endswith('.m3u8'): logger.debug("Can't stream HLS in non VLC mode: %s" % url) return None, None, None request = urllib2.Request(url, headers=req_headers) connection = self._streamReaderConnection = urllib2.urlopen( request, timeout=120) code = connection.getcode() if code not in (200, 206): logger.error("Failed to open video stream %s" % connection) return None, None, None FORWARD_HEADERS = [ 'Content-Range', 'Connection', 'Keep-Alive', 'Content-Type', 'Accept-Ranges', 'X-Content-Duration', 'Content-Length', ] SKIP_HEADERS = ['Server', 'Date'] response_headers = {} for k in connection.info().headers: if k.split(':')[0] not in (FORWARD_HEADERS + SKIP_HEADERS): logger.debug('NEW HEADERS: %s' % k.split(':')[0]) for h in FORWARD_HEADERS: if connection.info().getheader(h): response_headers[h] = connection.info().getheader(h) # self.connection.send_header(h, connection.info().getheader(h)) logger.debug('key=%s value=%s' % (h, connection.info().getheader(h))) with self._lock: self._streamReaderState = 2 self._lock.notifyAll() return connection, code, response_headers def startStreamReaderPT(self, url, cid, counter, req_headers=None): logger = logging.getLogger("StreamReaderPT") self._streamReaderState = 1 # current_req_headers = req_headers try: while True: data = None clients = counter.getClientsPT(cid) if not req_headers == self.req_headers: self.req_headers = req_headers connection, code, resp_headers = self.openStreamReaderPT( url, req_headers) if not connection: return for c in clients: try: c.headers_sent except: self.sendHeadersPT(c, code, resp_headers) c.headers_sent = True # logger.debug("i") try: data = connection.read(AceConfig.readchunksize) except: break # logger.debug("d: %s c:%s" % (data, clients)) if data and clients: with self._lock: if len(self._streamReaderQueue ) == AceConfig.readcachesize: self._streamReaderQueue.popleft() self._streamReaderQueue.append(data) for c in clients: try: c.addChunk(data, 5.0) except Queue.Full: if len(clients) > 1: logger.debug("Disconnecting client: %s" % str(c)) c.destroy() elif not clients: logger.debug( "All clients disconnected - closing video stream") break else: logger.warning("No data received") break except urllib2.URLError: logger.error("Failed to open video stream") logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) if counter.getClientsPT(cid): logger.error("Failed to read video stream") finally: self.closeStreamReader() with self._lock: self._streamReaderState = 3 self._lock.notifyAll() counter.deleteAll(cid)
class AceClient(object): def __init__(self, acehost, aceAPIport, aceHTTPport, acehostslist, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current video position self._position = None # Available video position (loaded data) self._position_last = None # Buffered video pieces self._position_buf = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Result for GETCID() self._cidresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Seekback seconds. self._seekback = AceConfig.videoseekback # Did we get START command again? For seekback. self._started_again = False self._idleSince = time.time() self._lock = threading.Condition(threading.Lock()) self._streamReaderConnection = None self._streamReaderState = None self._streamReaderQueue = deque() self._engine_version_code = 0; # Logger logger = logging.getLogger('AceClientimport tracebacknt_init') # Try to connect AceStream engine try: self._socket = telnetlib.Telnet(acehost, aceAPIport, connect_timeout) AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = acehost, aceAPIport, aceHTTPport logger.debug("Successfully connected to AceStream on %s:%d" % (acehost, aceAPIport)) except: for AceEngine in acehostslist: try: self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout) AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[0], AceEngine[1], AceEngine[2] logger.debug("Successfully connected to AceStream on %s:%d" % (AceEngine[0], AceEngine[1])) break except: logger.debug("The are no alive AceStream on %s:%d" % (AceEngine[0], AceEngine[1])) pass # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def reset(self): self._started_again = False self._idleSince = time.time() self._streamReaderState = None def _write(self, message): try: logger = logging.getLogger("AceClient_write") logger.debug(message) self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, seekback=0): self._product_key = product_key self._gender = gender self._age = age # Seekback seconds self._seekback = seekback # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def _getResult(self): # Logger try: result = self._result.get(timeout=self._resulttimeout) if not result: raise AceException("Result not received") except gevent.Timeout: raise AceException("Timeout") return result def START(self, datatype, value): ''' Start video method ''' if self._engine_version_code >= 3010500 and AceConfig.vlcuse: stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \ + ' transcode_mp3=' + str(AceConfig.transcode_mp3) \ + ' transcode_ac3=' + str(AceConfig.transcode_ac3) \ + ' preferred_audio_language=' + AceConfig.preferred_audio_language else: stream_type = 'output_format=http' self._urlresult = AsyncResult() self._write(AceMessage.request.START(datatype.upper(), value, stream_type)) self._getResult() def STOP(self): ''' Stop video method ''' if self._state is not None and self._state != '0': self._result = AsyncResult() self._write(AceMessage.request.STOP) self._getResult() def LOADASYNC(self, datatype, value): self._result = AsyncResult() self._write(AceMessage.request.LOADASYNC(datatype.upper(), random.randint(1, AceConfig.maxconns * 10000), value)) return self._getResult() def GETCONTENTINFO(self, datatype, value): dict = {'torrent': 'url', 'infohash':'infohash', 'raw':'data', 'pid':'content_id'} self.paramsdict={dict[datatype]:value,'developer_id':'0','affiliate_id':'0','zone_id':'0'} contentinfo = self.LOADASYNC(datatype, self.paramsdict) return contentinfo def GETCID(self, datatype, url): contentinfo = self.GETCONTENTINFO(datatype, url) self._cidresult = AsyncResult() self._write(AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0)) cid = self._cidresult.get(True, 5) return '' if not cid or cid == '' else cid[2:] def getUrl(self, timeout=30): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "Engine response time exceeded. GetURL timeout!" logger.error(errmsg) raise AceException(errmsg) def startStreamReader(self, url, cid, counter, req_headers=None): logger = logging.getLogger("StreamReader") logger.debug("Opening video stream: %s" % url) logger.debug("Get headers from client: %s" % req_headers) self._streamReaderState = 1 try: if url.endswith('.m3u8'): logger.debug("Can't stream HLS in non VLC mode: %s" % url) return None # Sending client hedaers to AceEngine if req_headers.has_key('range'): del req_headers['range'] connection = self._streamReaderConnection = requests.get(url, headers=req_headers, stream=True) if connection.status_code not in (200, 206): logger.error("Failed to open video stream %s" % url) return None with self._lock: self._streamReaderState = 2 self._lock.notifyAll() while True: data = None clients = counter.getClients(cid) try: data = connection.raw.read(AceConfig.readchunksize) except: break; if data and clients: with self._lock: if len(self._streamReaderQueue) == AceConfig.readchunksize: self._streamReaderQueue.popleft() self._streamReaderQueue.append(data) for c in clients: try: c.addChunk(data, 5.0) except Queue.Full: if len(clients) > 1: logger.debug("Disconnecting client: %s" % str(c)) c.destroy() elif not clients: logger.debug("All clients disconnected - closing video stream") break else: logger.warning("No data received") break except requests.exceptions.ConnectionError: logger.error("Failed to open video stream") logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) if counter.getClients(cid): logger.error("Failed to read video stream") finally: self.closeStreamReader() with self._lock: self._streamReaderState = 3 self._lock.notifyAll() counter.deleteAll(cid) def closeStreamReader(self): logger = logging.getLogger("StreamReader") c = self._streamReaderConnection if c: self._streamReaderConnection = None c.close() logger.debug("Video stream closed") self._streamReaderQueue.clear() def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' return self._resumeevent.wait(timeout=timeout) def pause(self): self._write(AceMessage.request.PAUSE) def play(self): self._write(AceMessage.request.PLAY) def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n") self._recvbuffer = self._recvbuffer.strip() logger.debug('<<< ' + self._recvbuffer) except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return if self._recvbuffer: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'version_code=' in self._recvbuffer: v = self._recvbuffer.find('version_code=') self._engine_version_code = int(self._recvbuffer[v+13:v+20]) if 'key=' in self._recvbuffer: self._request_key_begin = self._recvbuffer.find('key=') self._request_key = \ self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14] try: self._write(AceMessage.request.READY_key( self._request_key, self._product_key)) except requests.exceptions.ConnectionError as e: logger.error("Can't connect to keygen server! " + \ repr(e)) self._auth = False self._authevent.set() self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") self._auth = False self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): # LOADRESP _contentinfo_raw = self._recvbuffer.split()[2:] _contentinfo_raw = ' '.join(_contentinfo_raw) _contentinfo = json.loads(_contentinfo_raw) if _contentinfo.get('status') == 100: logger.error("LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: logger.debug("Content info: %s", _contentinfo) self._result.set(_contentinfo) elif self._recvbuffer.startswith(AceMessage.response.START): # START if not self._seekback or self._started_again or not self._recvbuffer.endswith(' stream=1'): # If seekback is disabled, we use link in first START command. # If seekback is enabled, we wait for first START command and # ignore it, then do seekback in first EVENT position command # AceStream sends us STOP and START again with new link. # We use only second link then. try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None else: logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS) # STOP elif self._recvbuffer.startswith(AceMessage.response.STOP): pass # SHUTDOWN elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return # AUTH elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() # GETUSERDATA elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") # LIVEPOS elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS): self._position = self._recvbuffer.split() self._position_last = self._position[2].split('=')[1] self._position_buf = self._position[9].split('=')[1] self._position = self._position[4].split('=')[1] if self._seekback and not self._started_again: self._write(AceMessage.request.LIVESEEK(str(int(self._position_last) - self._seekback))) logger.debug('Seeking back') self._started_again = True # DOWNLOADSTOP elif self._recvbuffer.startswith(AceMessage.response.DOWNLOADSTOP): self._state = self._recvbuffer.split()[1] # STATE elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] # STATUS elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to " + self._status) if self._status == 'main:err': logger.error( self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._status == 'main:idle': self._result.set(True) # PAUSE elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() # RESUME elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep() # PAUSE/RESUME delay self._resumeevent.set() # CID elif self._recvbuffer.startswith('##') or len(self._recvbuffer) == 0: self._cidresult.set(self._recvbuffer) logger.debug("CID: %s" %self._recvbuffer)
class TestMemoryCache(unittest.TestCase): """ Test description """ currentTempDir = "" def setUp(self): """ Setup """ # Reset counters Meters.reset() # Clear self.mem_cache = None self.callback_call = 0 self.callback_evicted = 0 self.callback_return = False self.evict_count = 0 self.evict_last_key = None self.evict_last_value = None def tearDown(self): """ Stop """ if self.mem_cache: logger.warning("Stopping mem_cache") self.mem_cache.stop_cache() self.mem_cache = None def watchdog_callback(self, evicted_count): """ Callback :param evicted_count: Evicted count :return: True if reschedule the callback, False otherwise """ logger.info("Called, evicted_count=%s", evicted_count) self.callback_call += 1 self.callback_evicted += evicted_count return self.callback_return def eviction_callback(self, k, v): """ Eviction callback :param k: key :param v: value """ self.evict_count += 1 self.evict_last_key = k self.evict_last_value = v def test_start_stop(self): """ Test. """ # Alloc self.mem_cache = MemoryCache() self.assertTrue(self.mem_cache._is_started) # Stop self.mem_cache.stop_cache() self.assertFalse(self.mem_cache._is_started) # Start self.mem_cache.start_cache() self.assertTrue(self.mem_cache._is_started) # Stop self.mem_cache.stop_cache() self.assertFalse(self.mem_cache._is_started) # Over self.mem_cache = None def test_size_handling(self): """ Test. """ # Alloc self.mem_cache = MemoryCache() # Put self.mem_cache.put("A", b"A1", 60000) self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 2) # Put again, not the same data self.mem_cache.put("A", b"A11", 60000) self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 3) # Put again, not the same data self.mem_cache.put("A", b"A", 60000) self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 1) # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_init_no_stat(self): """ Test. """ # Alloc m = MemoryCache() self.assertIsNotNone(m) def test_basic(self): """ Test. """ # Alloc self.mem_cache = MemoryCache() # Get : must return nothing o = self.mem_cache.get("not_found") self.assertIsNone(o) # Put self.mem_cache.put("keyA", b"valA", 60000) o = self.mem_cache.get("keyA") self.assertEqual(o, b"valA") o = self.mem_cache.get_raw("keyA") self.assertIsNotNone(o) self.assertIsInstance(o, tuple) self.assertEqual(o[1], b"valA") self.assertLessEqual(o[0] - SolBase.mscurrent(), 60000) logger.info("TTL approx=%s", o[0] - SolBase.mscurrent()) # Put with lower TTL : TTL MUST BE UPDATED self.mem_cache.put("keyA", b"valA", 30000) o = self.mem_cache.get("keyA") self.assertEqual(o, b"valA") o = self.mem_cache.get_raw("keyA") self.assertIsNotNone(o) self.assertIsInstance(o, tuple) self.assertEqual(o[1], b"valA") self.assertLessEqual(o[0] - SolBase.mscurrent(), 30000) logger.info("TTL approx=%s", o[0] - SolBase.mscurrent()) # Str put : must now be ok self.mem_cache.put("toto_str", "str_val", 1000) self.assertEqual(self.mem_cache.get("toto_str"), "str_val") # Non bytes,str injection (int) : must fail # noinspection PyBroadException,PyPep8 try: # noinspection PyTypeChecker self.mem_cache.put("toto", 12, 1000) self.fail("Must fail") except: pass # Non bytes injection : must fail # noinspection PyBroadException,PyPep8 try: # noinspection PyTypeChecker self.mem_cache.put("toto", u"unicode_buffer", 1000) self.fail("Must fail") except: pass # This MUST fail # noinspection PyBroadException try: # noinspection PyTypeChecker self.mem_cache.put(999, b"value", 60000) self.fail("Put a key as non bytes,str MUST fail") except Exception: pass # This MUST fail # noinspection PyBroadException try: # noinspection PyTypeChecker self.mem_cache.remove(999) self.fail("Remove a key as non bytes,str MUST fail") except Exception: pass # Put/Remove self.mem_cache.put("todel", b"value", 60000) self.assertEqual(self.mem_cache.get("todel"), b"value") self.mem_cache.remove("todel") self.assertEqual(self.mem_cache.get("todel"), None) # Put self.mem_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz", 60000) self.assertEqual( self.mem_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"), b"zzz") # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_basic_eviction_max_capacity_lru(self): """ Test :return: """ # Alloc self.mem_cache = MemoryCache(max_item=3, cb_evict=self.eviction_callback) # Put self.mem_cache.put("keyA", b"valA", 60000) self.mem_cache.put("keyB", b"valB", 60000) self.mem_cache.put("keyC", b"valC", 60000) # Use A and C => B becomes the older to be used self.mem_cache.get("keyA") self.mem_cache.get("keyC") # We are maxed (3 items) # Add D => B must be kicked self.mem_cache.put("keyD", b"valD", 60000) self.assertEqual(self.mem_cache.get("keyA"), b"valA") self.assertEqual(self.mem_cache.get("keyC"), b"valC") self.assertEqual(self.mem_cache.get("keyD"), b"valD") self.assertIsNone(self.mem_cache.get("keyB")) self.assertEqual(self.evict_count, 1) self.assertEqual(self.evict_last_key, "keyB") self.assertEqual(self.evict_last_value, b"valB") self.assertEqual(Meters.aig("mcs.cache_evict_lru_put"), 1) # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_basic_eviction_lru(self): """ Test :return: """ # Alloc self.mem_cache = MemoryCache(cb_evict=self.eviction_callback) # Put self.mem_cache.put("keyA", b"valA", 60000) self.mem_cache.put("keyB", b"valB", 60000) self.mem_cache.put("keyC", b"valC", 60000) # Evict LRU self.mem_cache._evict_key_lru() # A was put first => must be kicked self.assertIsNone(self.mem_cache.get("keyA")) self.assertEqual(self.mem_cache.get("keyB"), b"valB") self.assertEqual(self.mem_cache.get("keyC"), b"valC") self.assertEqual(self.evict_count, 1) self.assertEqual(self.evict_last_key, "keyA") self.assertEqual(self.evict_last_value, b"valA") # Evict LRU self.mem_cache._evict_key_lru() # B was put first => must be kicked self.assertIsNone(self.mem_cache.get("keyA")) self.assertIsNone(self.mem_cache.get("keyB")) self.assertEqual(self.mem_cache.get("keyC"), b"valC") self.assertEqual(self.evict_count, 2) self.assertEqual(self.evict_last_key, "keyB") self.assertEqual(self.evict_last_value, b"valB") # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_basic_eviction_with_get_lru(self): """ Test :return: """ # Alloc self.mem_cache = MemoryCache(cb_evict=self.eviction_callback) # Put self.mem_cache.put("keyA", b"valA", 60000) self.mem_cache.put("keyB", b"valB", 60000) # A was put first => must be kicked, but we GET IT => B must be kicked self.assertEqual(self.mem_cache.get("keyA"), b"valA") # Evict LRU self.mem_cache._evict_key_lru() # A was used more recently => B must be kicked self.assertIsNone(self.mem_cache.get("keyB")) self.assertEqual(self.mem_cache.get("keyA"), b"valA") self.assertEqual(self.evict_count, 1) self.assertEqual(self.evict_last_key, "keyB") self.assertEqual(self.evict_last_value, b"valB") # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_basic_eviction_with_get_ttl(self): """ Test :return: """ # Alloc self.mem_cache = MemoryCache(cb_evict=self.eviction_callback) # Put self.mem_cache.put("keyA", b"valA", 60000) self.mem_cache.put("keyB", b"valB", 500) logger.info("ms cur=%s", SolBase.mscurrent()) logger.info("A : %s", self.mem_cache.get_raw("keyA")) logger.info("B : %s", self.mem_cache.get_raw("keyB")) # Wait a bit SolBase.sleep(600) logger.info("ms after sleep=%s", SolBase.mscurrent()) # A : must be present # B : must be evicted (TTL elapsed) self.assertEqual(self.mem_cache.get("keyA"), b"valA") self.assertIsNone(self.mem_cache.get("keyB")) self.assertEqual(self.evict_count, 1) self.assertEqual(self.evict_last_key, "keyB") self.assertEqual(self.evict_last_value, b"valB") self.assertEqual(Meters.aig("mcs.cache_evict_ttl_get"), 1) # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_basic_eviction_with_watchdog_ttl(self): """ Test :return: """ # Go self.mem_cache = MemoryCache(watchdog_interval_ms=1000, cb_watchdog=self.watchdog_callback, cb_evict=self.eviction_callback) # Put self.mem_cache.put("keyA", b"valA", 60000) self.mem_cache.put("keyB", b"valB", 500) self.mem_cache.put("keyC", b"valC", 500) self.mem_cache.put("keyD", b"valD", 60000) logger.info("ms cur=%s", SolBase.mscurrent()) logger.info("A : %s", self.mem_cache.get_raw("keyA")) logger.info("B : %s", self.mem_cache.get_raw("keyB")) logger.info("C : %s", self.mem_cache.get_raw("keyC")) logger.info("D : %s", self.mem_cache.get_raw("keyD")) # Wait a bit ms_start = SolBase.mscurrent() while SolBase.msdiff(ms_start) < (1000.0 * 2.0): if self.callback_call > 0: break else: SolBase.sleep(10) logger.info("ms after wait=%s", SolBase.mscurrent()) logger.info("_hash_key = %s", self.mem_cache._hash_key) logger.info("_hash_context = %s", self.mem_cache._hash_context) # A : must be present # B : must be evicted (TTL elapsed, by watchdog) self.assertEqual(self.callback_call, 1) self.assertEqual(self.callback_evicted, 2) self.assertFalse("valB" in self.mem_cache._hash_key) self.assertFalse("valB" in self.mem_cache._hash_context) self.assertFalse("valC" in self.mem_cache._hash_key) self.assertFalse("valC" in self.mem_cache._hash_context) self.assertIsNone(self.mem_cache.get("keyB")) self.assertIsNone(self.mem_cache.get("keyC")) self.assertEqual(self.mem_cache.get("keyA"), b"valA") self.assertEqual(self.mem_cache.get("keyD"), b"valD") self.assertEqual(self.evict_count, 2) self.assertTrue(self.evict_last_key == "keyB" or self.evict_last_key == "keyC") self.assertTrue(self.evict_last_value == b"valB" or self.evict_last_value == b"valC") self.assertEqual(Meters.aig("mcs.cache_evict_ttl_watchdog"), 2) # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_watchdog_schedule(self): """ Test :return: """ # Continue callback loop self.callback_return = True # Go self.mem_cache = MemoryCache(watchdog_interval_ms=500, cb_watchdog=self.watchdog_callback) # Put self.mem_cache.put("keyA", b"valA", 60000) self.mem_cache.put("keyD", b"valD", 60000) # Wait a bit ms_start = SolBase.mscurrent() while SolBase.msdiff(ms_start) < (500.0 * 5.0): if self.callback_call >= 1: logger.info("Run 1 exit") break else: SolBase.sleep(1) logger.info("Run 1 done") self.assertEqual(self.callback_call, 1) self.assertEqual(self.callback_evicted, 0) # Wait a bit ms_start = SolBase.mscurrent() while SolBase.msdiff(ms_start) < (500.0 * 5.0): if self.callback_call >= 2: logger.info("Run 2 exit") break else: SolBase.sleep(1) logger.info("Run 2 done") self.assertGreaterEqual(self.callback_call, 2) self.assertEqual(self.callback_evicted, 0) # Stop self.mem_cache.stop_cache() self.mem_cache = None # Wait a bit SolBase.sleep(500 * 2) # Nothing must happen self.assertEqual(self.callback_call, 2) self.assertEqual(self.callback_evicted, 0) # ======================== # BENCH # ======================== def test_bench_greenlet_1_put_100_get_0_max_128000(self): """ Test :return: """ self._go_greenlet(1, 100, 0, 128000) def test_bench_greenlet_16_put_10_get_1_max_50(self): """ Test :return: """ self._go_greenlet(16, 1, 1, 128000, max_item=50) def test_bench_greenlet_16_put_10_get_1_maxsize_300bytes(self): """ Test :return: """ self._go_greenlet( 16, 1, 1, 128000, watchdog_interval_ms=500, max_item=50, max_bytes=300, max_single_item_bytes=50, purge_min_bytes=20, purge_min_count=5, ) def test_bench_greenlet_1_put_1_get_100_max_128000(self): """ Test :return: """ self._go_greenlet(1, 1, 100, 128000) def test_bench_greenlet_128_put_10_get_10_max_128000(self): """ Test :return: """ self._go_greenlet(128, 10, 10, 128000) def test_bench_greenlet_128_put_10_get_10_maxsize_10000(self): """ Test :return: """ self._go_greenlet( 128, 10, 10, 128000, watchdog_interval_ms=500, max_item=sys.maxsize, max_bytes=10000, max_single_item_bytes=50, purge_min_bytes=int(10000 / 2), purge_min_count=5000, ) def _go_greenlet(self, greenlet_count, put_count, get_count, bench_item_count, watchdog_interval_ms=60000, max_item=128000, max_bytes=32 * 1024 * 1024, max_single_item_bytes=1 * 1024 * 1024, purge_min_bytes=8 * 1024 * 1024, purge_min_count=1000): """ Doc :param greenlet_count: greenlet_count :param put_count: put_count :param get_count: get_count :param bench_item_count: bench_item_count :param watchdog_interval_ms: watchdog_interval_ms :param max_item: max_item :param max_bytes: max_bytes :param max_single_item_bytes: max_single_item_bytes :param purge_min_bytes: purge_min_bytes :param purge_min_count: purge_min_count """ g_event = None g_array = None try: # Settings g_count = greenlet_count g_ms = 10000 # Continue callback loop self.callback_return = True # Go self.mem_cache = MemoryCache( watchdog_interval_ms=watchdog_interval_ms, max_item=max_item, max_bytes=max_bytes, max_single_item_bytes=max_single_item_bytes, purge_min_bytes=purge_min_bytes, purge_min_count=purge_min_count) # Item count self.bench_item_count = bench_item_count self.bench_put_weight = put_count self.bench_get_weight = get_count self.bench_ttl_min_ms = 1000 self.bench_ttl_max_ms = int(g_ms / 2) # Go self.run_event = Event() self.exception_raised = 0 self.open_count = 0 self.thread_running = AtomicIntSafe() self.thread_running_ok = AtomicIntSafe() # Item per greenlet item_per_greenlet = self.bench_item_count / g_count # Signal self.gorun_event = Event() # Alloc greenlet g_array = list() g_event = list() for _ in range(0, g_count): greenlet = Greenlet() g_array.append(greenlet) g_event.append(Event()) # Run them cur_idx = 0 for idx in range(0, len(g_array)): greenlet = g_array[idx] event = g_event[idx] greenlet.spawn(self._run_cache_bench, event, cur_idx, cur_idx + item_per_greenlet) cur_idx += item_per_greenlet SolBase.sleep(0) # Signal self.gorun_event.set() # Wait a bit dt = SolBase.mscurrent() while SolBase.msdiff(dt) < g_ms: SolBase.sleep(500) # Stat ms = SolBase.msdiff(dt) sec = float(ms / 1000.0) total_put = Meters.aig("mcs.cache_put") per_sec_put = round(float(total_put) / sec, 2) total_get = Meters.aig("mcs.cache_get_hit") + Meters.aig( "mcs.cache_get_miss") per_sec_get = round(float(total_get) / sec, 2) logger.info( "Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s", self.open_count, self.thread_running.get(), self.thread_running_ok.get(), per_sec_put, per_sec_get, self.mem_cache) self.assertEqual(self.exception_raised, 0) # Over, signal logger.info("Signaling, count=%s", self.open_count) self.run_event.set() # Wait for g in g_event: g.wait(30.0) self.assertTrue(g.isSet()) g_event = None g_array = None # Log Meters.write_to_logger() finally: self.run_event.set() if g_event: for g in g_event: g.set() if g_array: for g in g_array: g.kill() if self.mem_cache: max_count = 0 total_size = 0 i = 0 for (k, v) in self.mem_cache._hash_key.items(): i += 1 total_size += len(k) + len(v[1]) if i < max_count: logger.info("%s => %s", k, v) self.assertEqual(total_size, self.mem_cache._current_data_bytes.get()) self.mem_cache.stop_cache() self.mem_cache = None def _run_cache_bench(self, event, idx_min, idx_max): """ Run :param idx_min: Index min :param idx_max: Index max """ idx_max -= 1 # Wait self.gorun_event.wait() # Go cur_count = 0 logger.debug("Entering now, idx_min=%s, idx_max=%s", idx_min, idx_max) self.thread_running.increment() self.thread_running_ok.increment() try: while not self.run_event.isSet(): cur_count += 1 try: cur_item = random.randint(idx_min, idx_max) s_cur_item = "%s" % cur_item b_cur_item = SolBase.unicode_to_binary(s_cur_item, "utf-8") cur_ttl = random.randint(self.bench_ttl_min_ms, self.bench_ttl_max_ms) for _ in range(0, self.bench_put_weight): self.mem_cache.put(s_cur_item, b_cur_item, cur_ttl) SolBase.sleep(0) for _ in range(0, self.bench_get_weight): v = self.mem_cache.get(s_cur_item) if v: self.assertEqual(v, b_cur_item) SolBase.sleep(0) except Exception as e: self.exception_raised += 1 logger.warning("Ex=%s", SolBase.extostr(e)) self.thread_running_ok.increment(-1) return finally: pass finally: self.assertGreater(cur_count, 0) logger.debug("Exiting") event.set() self.thread_running.increment(-1) # ===================================== # SIZE STUFF # ===================================== def test_size_limit_on_data_size_first(self): """ Test. """ # Alloc self.mem_cache = MemoryCache( max_bytes=5 * 10 * 2, max_single_item_bytes=6 * 2, purge_min_bytes=5 * 5 * 2, purge_min_count=2, max_item=sys.maxsize, ) # Put 10 items for i in range(10, 20): self.mem_cache.put("key" + str(i), SolBase.unicode_to_binary("val%s" % i, "utf-8"), 60000) logger.info("Cache=%s", self.mem_cache) # Must have all of them for i in range(10, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 10) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 10 * 2) # Then add a new one : we will over size the cache self.mem_cache.put("key" + str(99), SolBase.unicode_to_binary("val99", "utf-8"), 60000) # We must have evicted AT least : # 5*5 + 5 bytes => 5 items + the item added => 6 items # 2 items minimum # So 6 items : 10 to 15 must be evicted logger.info("Hash = %s", self.mem_cache._hash_key) for i in range(10, 16): self.assertIsNone(self.mem_cache.get("key" + str(i))) for i in range(16, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(self.mem_cache.get("key" + str(99)), SolBase.unicode_to_binary("val99", "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 5) self.assertEqual(len(self.mem_cache._hash_context), 5) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 5 * 2) # Try add a big one this time : must not be done (over limit) self.mem_cache.put("BIGDATA", b"aaaaaaaaaaaaaaaaaaaa", 60000) self.assertIsNone(self.mem_cache.get("BIGDATA")) # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_size_limit_on_min_item_to_evict_first(self): """ Test. """ # Alloc self.mem_cache = MemoryCache( max_bytes=5 * 10 * 2, max_single_item_bytes=6 * 2, purge_min_bytes=1 * 5, purge_min_count=6, max_item=sys.maxsize, ) # Put 10 items for i in range(10, 20): self.mem_cache.put("key" + str(i), SolBase.unicode_to_binary("val%s" % i, "utf-8"), 60000) logger.info("Cache=%s", self.mem_cache) # Must have all of them for i in range(10, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 10) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Then add a new one : we will over size the cache self.mem_cache.put("key" + str(99), SolBase.unicode_to_binary("val99", "utf-8"), 60000) # We must have evicted AT least : # 1*5 + 5 bytes => 1 items + the item added => 2 items # 6 items minimum # So : 10 to 15 must be evicted logger.info("Hash = %s", self.mem_cache._hash_key) for i in range(10, 16): self.assertIsNone(self.mem_cache.get("key" + str(i))) for i in range(16, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(self.mem_cache.get("key" + str(99)), SolBase.unicode_to_binary("val99", "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 5) self.assertEqual(len(self.mem_cache._hash_context), 5) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Try add a big one this time : must not be done (over limit) self.mem_cache.put("BIGDATA", b"aaaaaaaaaaaaaaaaaaaa", 60000) self.assertIsNone(self.mem_cache.get("BIGDATA")) self.mem_cache.put("aaaaaaaaaaaaaaaaaaaa", b"BIGKEY", 60000) self.assertIsNone(self.mem_cache.get("aaaaaaaaaaaaaaaaaaaa")) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_size_limit_cache_clear(self): """ Test. """ # Alloc self.mem_cache = MemoryCache( max_bytes=5 * 10 * 2, max_single_item_bytes=6 * 2, purge_min_bytes=1 * 5, purge_min_count=100, max_item=sys.maxsize, ) # Put 10 items for i in range(10, 20): self.mem_cache.put("key" + str(i), SolBase.unicode_to_binary("val%s" % i, "utf-8"), 60000) logger.info("Cache=%s", self.mem_cache) # Must have all of them for i in range(10, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 10) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Then add a new one : we will over size the cache self.mem_cache.put("key" + str(99), SolBase.unicode_to_binary("val99", "utf-8"), 60000) # We must have evicted AT least : # 1*5 + 5 bytes => 1 items + the item added => 2 items # 100 items minimum # So : All items must be kicked, will remain only the new one logger.info("Hash = %s", self.mem_cache._hash_key) self.assertEqual(self.mem_cache.get("key" + str(99)), SolBase.unicode_to_binary("val99", "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 1) self.assertEqual(len(self.mem_cache._hash_context), 1) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Stop self.mem_cache.stop_cache() self.mem_cache = None def test_size_limit_max_item_count(self): """ Test. """ # Alloc self.mem_cache = MemoryCache(max_bytes=sys.maxsize, max_single_item_bytes=6 * 2, purge_min_bytes=1 * 5, purge_min_count=0, max_item=10) # Put 10 items for i in range(10, 20): self.mem_cache.put("key" + str(i), SolBase.unicode_to_binary("val%s" % i, "utf-8"), 60000) logger.info("Cache=%s", self.mem_cache) # Must have all of them for i in range(10, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 10) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Then add a new one : we will over size the cache self.mem_cache.put("key" + str(99), SolBase.unicode_to_binary("val99", "utf-8"), 60000) # We must have evicted only first added logger.info("Hash = %s", self.mem_cache._hash_key) self.assertIsNone(self.mem_cache.get("key" + str(10))) for i in range(11, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(self.mem_cache.get("key" + str(99)), SolBase.unicode_to_binary("val99", "utf-8")) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Stop self.mem_cache.stop_cache() self.mem_cache = None
def _remove_client(self, client_id, evt): """ Remove a client. Return a TcpServerClientContext upon success, :param client_id: The client id. :type client_id: int :param evt: Event to signal :type evt: gevent.Event, None :return The removed TcpServerClientContext or None upon failure. :rtype None,pysoltcp.tcpserver.clientcontext.TcpServerClientContext.TcpServerClientContext """ logger.debug("entering, client_id=%s", client_id) try: with self._client_connected_hash_lock: # Check if client_id not in self._client_connected_hash: # Note : This may occurs in some conditions logger.debug("client_id not hashed, id=%s", client_id) Meters.aii("tcp.server.client_remove_nothashed") return None # Get (direct, we are already in lock) old_client = self._get_client_fromid(client_id) # Remove from hashmap logger.debug("un-hashing, client_id=%s", client_id) del (self._client_connected_hash[client_id]) # ------------------------------ # Out of lock : call async : BUSINESS # ------------------------------ try: local_evt = Event() g1 = gevent.spawn(self._remove_client_stop_business, old_client, local_evt) SolBase.sleep(0) local_evt.wait(self._tcp_server_config.stop_client_timeout_ms / 1000.0) if not local_evt.isSet(): # Flush out warning s = "Greenlet dump, g={0}, frame={1}".format( g1, ''.join(traceback.format_stack(g1.gr_frame))) # Cleanup s = s.replace("\n", " # ") while s.find(" ") >= 0: s = s.replace(" ", " ") # Error logs logger.error( "Timeout in _remove_client_stop_business client_id=%s, stack=%s", client_id, s) # Kill g1.kill(block=True) # Stat Meters.aii("tcp.server.client_remove_timeout_business") except Exception as e: logger.warning( "Exception in _remove_client_stop_business client_id=%s, ex=%s", client_id, SolBase.extostr(e)) # ------------------------------ # Out of lock : call async : INTERNAL # ------------------------------ try: local_evt = Event() g2 = gevent.spawn(self._remove_client_stop_internal, old_client, local_evt) SolBase.sleep(0) local_evt.wait(self._tcp_server_config.stop_client_timeout_ms / 1000.0) if not local_evt.isSet(): # Flush out warning s = "Greenlet dump, g={0}, frame={1}".format( g2, ''.join(traceback.format_stack(g2.gr_frame))) # Cleanup s = s.replace("\n", " # ") while s.find(" ") >= 0: s = s.replace(" ", " ") # Error logs logger.error( "Timeout in _remove_client_stop_internal client_id=%s, stack=%s", client_id, s) # Kill g2.kill(block=True) # Stat Meters.aii("tcp.server.client_remove_timeout_internal") except Exception as e: logger.warning( "Exception in _remove_client_stop_internal client_id=%s, ex=%s", client_id, SolBase.extostr(e)) # Statistics Meters.aii("tcp.server.client_connected", -1) Meters.aii("tcp.server.client_remove_count") # Log logger.debug("client removed, id=%s, addr=%s", old_client.get_client_id(), old_client.get_client_addr()) return old_client except Exception as e: # Error logger.warning("Exception, ex=%s", SolBase.extostr(e)) # Statistics Meters.aii("tcp.server.client_remove_exception") return None finally: if evt: evt.set()
class AceClient(object): def __init__(self, host, port, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current video position self._position = None # Available video position (loaded data) self._position_last = None # Buffered video pieces self._position_buf = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Seekback seconds. self._seekback = 0 # Did we get START command again? For seekback. self._started_again = False # Logger logger = logging.getLogger('AceClient_init') try: self._socket = telnetlib.Telnet(host, port, connect_timeout) logger.info("Successfully connected with Ace!") except Exception as e: raise AceException( "Socket creation error! Ace is not running? " + repr(e)) # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def _write(self, message): try: self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0, seekback=0): self._product_key = product_key self._gender = gender self._age = age # PAUSE/RESUME delay self._pausedelay = pause_delay # Seekback seconds self._seekback = seekback # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def _getResult(self): # Logger logger = logging.getLogger("AceClient_START") try: result = self._result.get(timeout=self._resulttimeout) if not result: errmsg = "START error!" logger.error(errmsg) raise AceException(errmsg) except gevent.Timeout: errmsg = "START timeout!" logger.error(errmsg) raise AceException(errmsg) return result def START(self, datatype, value): ''' Start video method ''' self._result = AsyncResult() self._urlresult = AsyncResult() self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value)) contentinfo = self._getResult() self._write(AceMessage.request.START(datatype.upper(), value)) self._getResult() return contentinfo def getUrl(self, timeout=40): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "getURL timeout!" logger.error(errmsg) raise AceException(errmsg) def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' return self._resumeevent.wait(timeout=timeout) def pause(self): self._write(AceMessage.request.PAUSE) def play(self): self._write(AceMessage.request.PLAY) def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n") self._recvbuffer = self._recvbuffer.strip() #logger.debug('<<< ' + self._recvbuffer) except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return if self._recvbuffer: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'key=' in self._recvbuffer: self._request_key_begin = self._recvbuffer.find('key=') self._request_key = \ self._recvbuffer[self._request_key_begin+4:self._request_key_begin+14] try: self._write(AceMessage.request.READY_key( self._request_key, self._product_key)) except: self._auth = False self._authevent.set() self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") self._auth = False self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): # LOADRESP _contentinfo_raw = self._recvbuffer.split()[2:] _contentinfo_raw = ' '.join(_contentinfo_raw) _contentinfo = json.loads(_contentinfo_raw) if _contentinfo.get('status') == 100: logger.error("LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: logger.debug("Content info: %s", _contentinfo) _filename = urllib2.unquote(_contentinfo.get('files')[0][0]) self._result.set(_filename) elif self._recvbuffer.startswith(AceMessage.response.START): # START if not self._seekback or (self._seekback and self._started_again): # If seekback is disabled, we use link in first START command. # If seekback is enabled, we wait for first START command and # ignore it, then do seeback in first EVENT position command # AceStream sends us STOP and START again with new link. # We use only second link then. try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None elif self._recvbuffer.startswith(AceMessage.response.STOP): pass elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS): self._position = self._recvbuffer.split() self._position_last = self._position[2].split('=')[1] self._position_buf = self._position[9].split('=')[1] self._position = self._position[4].split('=')[1] # logger.debug('Current position/last/buf: %s/%s/%s' % (self._position, self._position_last, self._position_buf)) if self._seekback and not self._started_again: self._write(AceMessage.request.SEEK(str(int(self._position_last) - \ self._seekback))) logger.debug('Seeking back') self._started_again = True elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to {0} data {1}".format(self._status, repr(self._recvbuffer))) if self._status == 'main:dl': logger.debug("progress - speed {3}kb/s peers {6}".format(*self._recvbuffer.split(';'))) elif self._status == 'main:prebuf' or self._status == 'main:buf': logger.debug("progress {1}% speed {5}kb/s peers {8}".format(*self._recvbuffer.split(';'))) elif self._status == 'main:err': logger.error( self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep(self._pausedelay) self._resumeevent.set()
class TestRedisCache(unittest.TestCase): """ Test description """ currentTempDir = "" def setUp(self): """ Setup """ # Reset counters Meters.reset() # Clear self.redis_cache = None # Key prefix self.key_prefix = "rk_" + str(int(SolBase.mscurrent())) + "_" # Temp redis : clear ALL r = redis.Redis() r.flushall() del r def tearDown(self): """ Stop """ if self.redis_cache: logger.warning("Stopping redis_cache") self.redis_cache.stop_cache() self.redis_cache = None # Temp redis : clear ALL r = redis.Redis() r.flushall() del r def test_start_stop(self): """ Test. """ # Alloc self.redis_cache = RedisCache() self.assertTrue(self.redis_cache._is_started) # Stop self.redis_cache.stop_cache() self.assertFalse(self.redis_cache._is_started) # Start self.redis_cache.start_cache() self.assertTrue(self.redis_cache._is_started) # Stop self.redis_cache.stop_cache() self.assertFalse(self.redis_cache._is_started) # Over self.redis_cache = None def test_init_no_stat(self): """ Test. """ # Alloc m = RedisCache() self.assertIsNotNone(m) m.stop_cache() def test_basic(self): """ Test. """ # Alloc self.redis_cache = RedisCache() # Get : must return nothing o = self.redis_cache.get(self.key_prefix + "not_found") self.assertIsNone(o) # Put self.redis_cache.put(self.key_prefix + "keyA", b"valA", 60000) o = self.redis_cache.get(self.key_prefix + "keyA") self.assertEqual(o, b"valA") # Delete self.redis_cache.remove(self.key_prefix + "keyA") self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyA")) # Non bytes injection : must fail # noinspection PyBroadException,PyPep8 try: # noinspection PyTypeChecker self.redis_cache.put(self.key_prefix + "toto", 12, 1000) self.fail("Must fail") except: pass # Non bytes injection : must fail # noinspection PyBroadException,PyPep8 try: # noinspection PyTypeChecker self.redis_cache.put(self.key_prefix + "toto", u"unicode_buffer", 1000) self.fail("Must fail") except: pass # This MUST fail # noinspection PyBroadException try: # noinspection PyTypeChecker self.redis_cache.put(999, b"value", 60000) self.fail("Put a key as non bytes,str MUST fail") except Exception: pass # This MUST fail # noinspection PyBroadException try: # noinspection PyTypeChecker self.redis_cache.remove(999) self.fail("Remove a key as non bytes,str MUST fail") except Exception: pass # Put/Remove self.redis_cache.put(self.key_prefix + "todel", b"value", 60000) self.assertEqual(self.redis_cache.get(self.key_prefix + "todel"), b"value") self.redis_cache.remove(self.key_prefix + "todel") self.assertEqual(self.redis_cache.get(self.key_prefix + "todel"), None) # Put self.redis_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz", 60000) self.assertEqual( self.redis_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"), b"zzz") # Stop self.redis_cache.stop_cache() self.redis_cache = None def test_basic_external_redis(self): """ Test. """ # External r = Redis() # Alloc self.redis_cache = RedisCache( pool_read_d=None, pool_write_d=None, redis_read_client=r, redis_write_client=r, ) # Get : must return nothing o = self.redis_cache.get(self.key_prefix + "not_found") self.assertIsNone(o) # Put self.redis_cache.put(self.key_prefix + "keyA", b"valA", 60000) o = self.redis_cache.get(self.key_prefix + "keyA") self.assertEqual(o, b"valA") # Delete self.redis_cache.remove(self.key_prefix + "keyA") self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyA")) # Non bytes injection : must fail # noinspection PyBroadException,PyPep8 try: # noinspection PyTypeChecker self.redis_cache.put(self.key_prefix + "toto", 12, 1000) self.fail("Must fail") except: pass # Non bytes injection : must fail # noinspection PyBroadException,PyPep8 try: # noinspection PyTypeChecker self.redis_cache.put(self.key_prefix + "toto", u"unicode_buffer", 1000) self.fail("Must fail") except: pass # This MUST fail # noinspection PyBroadException try: # noinspection PyTypeChecker self.redis_cache.put(999, b"value", 60000) self.fail("Put a key as non bytes,str MUST fail") except Exception: pass # This MUST fail # noinspection PyBroadException try: # noinspection PyTypeChecker self.redis_cache.remove(999) self.fail("Remove a key as non bytes,str MUST fail") except Exception: pass # Put/Remove self.redis_cache.put(self.key_prefix + "todel", b"value", 60000) self.assertEqual(self.redis_cache.get(self.key_prefix + "todel"), b"value") self.redis_cache.remove(self.key_prefix + "todel") self.assertEqual(self.redis_cache.get(self.key_prefix + "todel"), None) # Put self.redis_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz", 60000) self.assertEqual( self.redis_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"), b"zzz") # Stop self.redis_cache.stop_cache() self.redis_cache = None def test_basic_ttl(self): """ Test :return: """ # Alloc self.redis_cache = RedisCache() # Put self.redis_cache.put(self.key_prefix + "keyA", b"valA", 60000) self.redis_cache.put(self.key_prefix + "keyB", b"valB", 1000) logger.info("ms cur=%s", SolBase.mscurrent()) logger.info("A : %s", self.redis_cache.get(self.key_prefix + "keyA")) logger.info("B : %s", self.redis_cache.get(self.key_prefix + "keyB")) # Wait a bit SolBase.sleep(2000) logger.info("ms after sleep=%s", SolBase.mscurrent()) # A : must be present # B : must be evicted (TTL elapsed) self.assertEqual(self.redis_cache.get(self.key_prefix + "keyA"), b"valA") self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyB")) self.assertEqual(Meters.aig("rcs.cache_put"), 2) self.assertEqual(Meters.aig("rcs.cache_get_hit"), 3) self.assertEqual(Meters.aig("rcs.cache_get_miss"), 1) # Stop self.redis_cache.stop_cache() self.redis_cache = None def test_max_item_size(self): """ Test :return: """ # Alloc self.redis_cache = RedisCache(max_single_item_bytes=2) # Put self.assertTrue( self.redis_cache.put(self.key_prefix + "keyA", b"aa", 60000)) self.assertFalse( self.redis_cache.put(self.key_prefix + "keyB", b"aaa", 60000)) logger.info("ms cur=%s", SolBase.mscurrent()) logger.info("A : %s", self.redis_cache.get(self.key_prefix + "keyA")) logger.info("B : %s", self.redis_cache.get(self.key_prefix + "keyB")) # A : must be present # B : must be out of cache self.assertEqual(self.redis_cache.get(self.key_prefix + "keyA"), b"aa") self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyB")) self.assertEqual(Meters.aig("rcs.cache_put"), 1) self.assertEqual(Meters.aig("rcs.cache_get_hit"), 2) self.assertEqual(Meters.aig("rcs.cache_get_miss"), 2) # Stop self.redis_cache.stop_cache() self.redis_cache = None # ======================== # BENCH # ======================== def test_bench_greenlet_1_put_100_get_0_max_128000(self): """ Test :return: """ self._go_greenlet(1, 100, 0, 128000) def test_bench_greenlet_16_put_10_get_1_max_128000(self): """ Test :return: """ self._go_greenlet(16, 1, 1, 128000) def test_bench_greenlet_128_put_10_get_10_max_128000(self): """ Test :return: """ self._go_greenlet(128, 10, 10, 128000) def _go_greenlet(self, greenlet_count, put_count, get_count, bench_item_count): """ Doc :param greenlet_count: greenlet_count :param put_count: put_count :param get_count: get_count :param bench_item_count: bench_item_count """ g_event = None g_array = None try: # Settings g_count = greenlet_count g_ms = 10000 # Continue callback loop self.callback_return = True # Go self.redis_cache = RedisCache() # Item count self.bench_item_count = bench_item_count self.bench_put_weight = put_count self.bench_get_weight = get_count self.bench_ttl_min_ms = 1000 self.bench_ttl_max_ms = int(g_ms / 2) # Go self.run_event = Event() self.exception_raised = 0 self.open_count = 0 self.thread_running = AtomicIntSafe() self.thread_running_ok = AtomicIntSafe() # Item per greenlet item_per_greenlet = self.bench_item_count / g_count # Signal self.gorun_event = Event() # Alloc greenlet g_array = list() g_event = list() for _ in range(0, g_count): greenlet = Greenlet() g_array.append(greenlet) g_event.append(Event()) # Run them cur_idx = 0 for idx in range(0, len(g_array)): greenlet = g_array[idx] event = g_event[idx] greenlet.spawn(self._run_cache_bench, event, cur_idx, cur_idx + item_per_greenlet) cur_idx += item_per_greenlet SolBase.sleep(0) # Signal self.gorun_event.set() # Wait a bit dt = SolBase.mscurrent() while SolBase.msdiff(dt) < g_ms: SolBase.sleep(500) # Stat ms = SolBase.msdiff(dt) sec = float(ms / 1000.0) total_put = Meters.aig("rcs.cache_put") per_sec_put = round(float(total_put) / sec, 2) total_get = Meters.aig("rcs.cache_get_hit") + Meters.aig( "rcs.cache_get_miss") per_sec_get = round(float(total_get) / sec, 2) logger.info( "Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s", self.open_count, self.thread_running.get(), self.thread_running_ok.get(), per_sec_put, per_sec_get, self.redis_cache) self.assertEqual(self.exception_raised, 0) # Over, signal logger.info("Signaling, count=%s", self.open_count) self.run_event.set() # Wait for g in g_event: g.wait(30.0) self.assertTrue(g.isSet()) g_event = None g_array = None # Log Meters.write_to_logger() finally: self.run_event.set() if g_event: for g in g_event: g.set() if g_array: for g in g_array: g.kill() if self.redis_cache: self.redis_cache.stop_cache() self.redis_cache = None def _run_cache_bench(self, event, idx_min, idx_max): """ Run :param idx_min: Index min :param idx_max: Index max """ idx_max -= 1 # Wait self.gorun_event.wait() # Go cur_count = 0 logger.debug("Entering now, idx_min=%s, idx_max=%s", idx_min, idx_max) self.thread_running.increment() self.thread_running_ok.increment() try: while not self.run_event.isSet(): cur_count += 1 try: cur_item = random.randint(idx_min, idx_max) s_cur_item = "%s" % cur_item b_cur_item = SolBase.unicode_to_binary(s_cur_item, "utf-8") cur_ttl = random.randint(self.bench_ttl_min_ms, self.bench_ttl_max_ms) for _ in range(0, self.bench_put_weight): self.redis_cache.put(s_cur_item, b_cur_item, cur_ttl) SolBase.sleep(0) for _ in range(0, self.bench_get_weight): v = self.redis_cache.get(s_cur_item) if v: self.assertEqual(v, b_cur_item) SolBase.sleep(0) except Exception as e: self.exception_raised += 1 logger.warning("Ex=%s", SolBase.extostr(e)) self.thread_running_ok.increment(-1) return finally: pass finally: self.assertGreater(cur_count, 0) logger.debug("Exiting") event.set() self.thread_running.increment(-1)
class Config(object): cmds = { 'heartbeat_interval': (int, None), 'command_port': (int, None), 'data_port': (int, None), 'pid_dir': (str, DEFAULT_PID_DIR), 'log_level': (str, DEFAULT_LOG_LEVEL), 'log_config': (str, None), 'antelope_orb_name': (str, None), 'antelope_orb_select': (str, None), 'antelope_orb_reject': (str, None), 'antelope_orb_after': (float, -1), } # TODO: What about command_port? CONFIG_DEPS = set([ 'heartbeat_interval', 'data_port', 'antelope_orb_name', ]) # When one of these changes, it signals the data server to restart DATASERVER_DEPS = [ 'data_port', 'antelope_orb_name', 'antelope_orb_select', 'antelope_orb_reject', 'antelope_orb_after', ] def setval(self, name, val, *args, **kwargs): setattr(self, name, val) def __init__(self, options, cmdproc): self.configuredevent = Event() self.dataserverconfigupdate = Event() self.heartbeatactive = Event() for name, (converter, default) in self.cmds.iteritems(): self.cmdproc = cmdproc # Initialize attr with default val setattr(self, name, default) # Create command to set attr # cmdserver sends sock after val; will that mess this up? setval = partial(self.setval, name) cmdproc.setCmd(name, converter, setval) # Set a default loglevel? # update from config file if hasattr(options, 'conffile') and options.conffile is not None: self.readConfig(options.conffile) # update from command line if hasattr(options, 'verbose') and options.verbose is True: self.log_level = 'debug' if hasattr(options, 'command_port') and options.command_port is not None: self.command_port = options.command_port def readConfig(self, conffile): with open(conffile, 'rU') as file: for line in file: self.cmdproc.processCmd(line) def __setattr__(self, name, value): super(Config, self).__setattr__(name, value) if not self.configuredevent.isSet(): configured_attrs = set() for attr in self.CONFIG_DEPS: if hasattr(self, attr) and getattr(self, attr) is not None: configured_attrs.add(attr) if configured_attrs == self.CONFIG_DEPS: self.configuredevent.set() if name in self.DATASERVER_DEPS: self.dataserverconfigupdate.set() self.dataserverconfigupdate.clear() @property def heartbeat_interval(self): return self._heartbeat_interval @heartbeat_interval.setter def heartbeat_interval(self, value): self._heartbeat_interval = value if value > 0: self.heartbeatactive.set() else: self.heartbeatactive.clear() @property def log_level(self): return self._log_level @log_level.setter def log_level(self, val): levels = { 'error': logging.ERROR, 'warn': logging.WARNING, 'info': logging.INFO, 'debug': logging.DEBUG, 'mesg': logging.DEBUG, } try: level = levels[val] except KeyError: log.error("Unknown logging level %s" % val) else: self._log_level = val logging.getLogger().setLevel(level) @property def log_config(self): return self._log_config @log_config.setter def log_config(self, val): try: ooi.logging.config.add_configuration(val) self._log_config = val except Exception: log.error("Failed to read log config '%s'" % val, exc_info=True)
class AceClient(object): def __init__(self, host, port, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current video position self._position = None # Available video position (loaded data) self._position_last = None # Buffered video pieces self._position_buf = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Result for GETCID() self._cidresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Seekback seconds. self._seekback = 0 # Did we get START command again? For seekback. self._started_again = False self._idleSince = time.time() self._lock = threading.Condition(threading.Lock()) self._streamReaderConnection = None self._streamReaderState = None self._streamReaderQueue = deque() self._engine_version_code = 0; # Logger logger = logging.getLogger('AceClieimport tracebacknt_init') try: self._socket = telnetlib.Telnet(host, port, connect_timeout) logger.info("Successfully connected with Ace!") except Exception as e: raise AceException( "Socket creation error! Ace is not running? " + repr(e)) # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def reset(self): self._started_again = False self._idleSince = time.time() self._streamReaderState = None def _write(self, message): try: logger = logging.getLogger("AceClient_write") logger.debug(message) self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0, seekback=0): self._product_key = product_key self._gender = gender self._age = age # PAUSE/RESUME delay self._pausedelay = pause_delay # Seekback seconds self._seekback = seekback # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def _getResult(self): # Logger try: result = self._result.get(timeout=self._resulttimeout) if not result: raise AceException("Result not received") except gevent.Timeout: raise AceException("Timeout") return result def START(self, datatype, value): ''' Start video method ''' stream_type = 'output_format=http' if self._engine_version_code >= 3010500 and not AceConfig.vlcuse else '' self._urlresult = AsyncResult() self._write(AceMessage.request.START(datatype.upper(), value, stream_type)) self._getResult() def STOP(self): ''' Stop video method ''' if self._state is not None and self._state != '0': self._result = AsyncResult() self._write(AceMessage.request.STOP) self._getResult() def LOADASYNC(self, datatype, url): self._result = AsyncResult() self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, {'url': url})) return self._getResult() def GETCID(self, datatype, url): contentinfo = self.LOADASYNC(datatype, url) self._cidresult = AsyncResult() self._write(AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0)) cid = self._cidresult.get(True, 5) return '' if not cid or cid == '' else cid[2:] def GETCONTENTINFO(self, datatype, url): contentinfo = self.LOADASYNC(datatype, url) return contentinfo def getUrl(self, timeout=40): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "getURL timeout!" logger.error(errmsg) raise AceException(errmsg) def startStreamReader(self, url, cid, counter): logger = logging.getLogger("StreamReader") self._streamReaderState = 1 logger.debug("Opening video stream: %s" % url) try: connection = self._streamReaderConnection = urllib2.urlopen(url) if url.endswith('.m3u8'): logger.debug("Can't stream HLS in non VLC mode: %s" % url) return if connection.getcode() != 200: logger.error("Failed to open video stream %s" % connection) return with self._lock: self._streamReaderState = 2 self._lock.notifyAll() while True: data = None clients = counter.getClients(cid) try: data = connection.read(AceConfig.readchunksize) except: break; if data and clients: with self._lock: if len(self._streamReaderQueue) == AceConfig.readcachesize: self._streamReaderQueue.popleft() self._streamReaderQueue.append(data) for c in clients: try: c.addChunk(data, 5.0) except Queue.Full: if len(clients) > 1: logger.debug("Disconnecting client: %s" % str(c)) c.destroy() elif not clients: logger.debug("All clients disconnected - closing video stream") break else: logger.warning("No data received") break except urllib2.URLError: logger.error("Failed to open video stream") logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) if counter.getClients(cid): logger.error("Failed to read video stream") finally: self.closeStreamReader() with self._lock: self._streamReaderState = 3 self._lock.notifyAll() counter.deleteAll(cid) def closeStreamReader(self): logger = logging.getLogger("StreamReader") c = self._streamReaderConnection if c: self._streamReaderConnection = None c.close() logger.debug("Video stream closed") self._streamReaderQueue.clear() def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' return self._resumeevent.wait(timeout=timeout) def pause(self): self._write(AceMessage.request.PAUSE) def play(self): self._write(AceMessage.request.PLAY) def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n") self._recvbuffer = self._recvbuffer.strip() # logger.debug('<<< ' + self._recvbuffer) except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return if self._recvbuffer: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'version_code=' in self._recvbuffer: v = self._recvbuffer.find('version_code=') self._engine_version_code = int(self._recvbuffer[v + 13:v + 20]) if 'key=' in self._recvbuffer: self._request_key_begin = self._recvbuffer.find('key=') self._request_key = \ self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14] try: self._write(AceMessage.request.READY_key( self._request_key, self._product_key)) except urllib2.URLError as e: logger.error("Can't connect to keygen server! " + \ repr(e)) self._auth = False self._authevent.set() self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") self._auth = False self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): # LOADRESP _contentinfo_raw = self._recvbuffer.split()[2:] _contentinfo_raw = ' '.join(_contentinfo_raw) _contentinfo = json.loads(_contentinfo_raw) if _contentinfo.get('status') == 100: logger.error("LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: logger.debug("Content info: %s", _contentinfo) self._result.set(_contentinfo) elif self._recvbuffer.startswith(AceMessage.response.START): # START if not self._seekback or self._started_again or not self._recvbuffer.endswith(' stream=1'): # If seekback is disabled, we use link in first START command. # If seekback is enabled, we wait for first START command and # ignore it, then do seeback in first EVENT position command # AceStream sends us STOP and START again with new link. # We use only second link then. try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None else: logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS) elif self._recvbuffer.startswith(AceMessage.response.STOP): pass elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS): self._position = self._recvbuffer.split() self._position_last = self._position[2].split('=')[1] self._position_buf = self._position[9].split('=')[1] self._position = self._position[4].split('=')[1] if self._seekback and not self._started_again: self._write(AceMessage.request.SEEK(str(int(self._position_last) - \ self._seekback))) logger.debug('Seeking back') self._started_again = True elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to " + self._status) if self._status == 'main:err': logger.error( self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._status == 'main:idle': self._result.set(True) elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep(self._pausedelay) self._resumeevent.set() elif self._recvbuffer.startswith('##') or len(self._recvbuffer) == 0: self._cidresult.set(self._recvbuffer) logger.debug("CID: %s" % self._recvbuffer) def sendHeadersPT(self, client, code, headers): client.handler.send_response(code) for key, value in headers.items(): client.handler.send_header(key, value) client.handler.end_headers() def openStreamReaderPT(self, url, req_headers): logger = logging.getLogger("openStreamReaderPT") logger.debug("Opening video stream: %s" % url) logger.debug("headers: %s" % req_headers) if url.endswith('.m3u8'): logger.debug("Can't stream HLS in non VLC mode: %s" % url) return None, None, None request = urllib2.Request(url, headers=req_headers) connection = self._streamReaderConnection = urllib2.urlopen(request, timeout=120) code = connection.getcode() if code not in (200, 206): logger.error("Failed to open video stream %s" % connection) return None, None, None FORWARD_HEADERS = ['Content-Range', 'Connection', 'Keep-Alive', 'Content-Type', 'Accept-Ranges', 'X-Content-Duration', 'Content-Length', ] SKIP_HEADERS = ['Server', 'Date'] response_headers = {} for k in connection.info().headers: if k.split(':')[0] not in (FORWARD_HEADERS + SKIP_HEADERS): logger.debug('NEW HEADERS: %s' % k.split(':')[0]) for h in FORWARD_HEADERS: if connection.info().getheader(h): response_headers[h] = connection.info().getheader(h) # self.connection.send_header(h, connection.info().getheader(h)) logger.debug('key=%s value=%s' % (h, connection.info().getheader(h))) with self._lock: self._streamReaderState = 2 self._lock.notifyAll() return connection, code, response_headers def startStreamReaderPT(self, url, cid, counter, req_headers=None): logger = logging.getLogger("StreamReaderPT") self._streamReaderState = 1 # current_req_headers = req_headers try: while True: data = None clients = counter.getClientsPT(cid) if not req_headers == self.req_headers: self.req_headers = req_headers connection, code, resp_headers = self.openStreamReaderPT(url, req_headers) if not connection: return for c in clients: try: c.headers_sent except: self.sendHeadersPT(c, code, resp_headers) c.headers_sent = True # logger.debug("i") try: data = connection.read(AceConfig.readchunksize) except: break; # logger.debug("d: %s c:%s" % (data, clients)) if data and clients: with self._lock: if len(self._streamReaderQueue) == AceConfig.readcachesize: self._streamReaderQueue.popleft() self._streamReaderQueue.append(data) for c in clients: try: c.addChunk(data, 5.0) except Queue.Full: if len(clients) > 1: logger.debug("Disconnecting client: %s" % str(c)) c.destroy() elif not clients: logger.debug("All clients disconnected - closing video stream") break else: logger.warning("No data received") break except urllib2.URLError: logger.error("Failed to open video stream") logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) if counter.getClientsPT(cid): logger.error("Failed to read video stream") finally: self.closeStreamReader() with self._lock: self._streamReaderState = 3 self._lock.notifyAll() counter.deleteAll(cid)
class AceClient(object): def __init__(self, acehostslist, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current video position self._position = None # Available video position (loaded data) self._position_last = None # Buffered video pieces self._position_buf = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Result for GETCID() self._cidresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Seekback seconds. self._seekback = AceConfig.videoseekback # Did we get START command again? For seekback. self._started_again = False self._idleSince = time.time() self._lock = threading.Condition(threading.Lock()) self._streamReaderConnection = None self._streamReaderState = None self._streamReaderQueue = deque() self._engine_version_code = 0 # Logger logger = logging.getLogger('AceClientimport tracebacknt_init') # Try to connect AceStream engine for AceEngine in acehostslist: try: self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout) AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[ 0], AceEngine[1], AceEngine[2] logger.debug("Successfully connected to AceStream on %s:%d" % (AceEngine[0], AceEngine[1])) break except: logger.debug("The are no alive AceStream on %s:%d" % (AceEngine[0], AceEngine[1])) pass # Spawning recvData greenlet if self._socket: gevent.spawn(self._recvData) gevent.sleep() else: logger.error("The are no alive AceStream Engines found") return def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): return # Already in the middle of destroying logger = logging.getLogger("AceClient_destroy") # Logger self._resumeevent.set( ) # We should resume video to prevent read greenlet deadlock self._urlresult.set() # And to prevent getUrl deadlock # Trying to disconnect try: logger.debug("Destroying AceStream client ...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: pass # Ignore exceptions on destroy finally: self._shuttingDown.set() def reset(self): self._started_again = False self._idleSince = time.time() self._streamReaderState = None def _write(self, message): try: logger = logging.getLogger("AceClient_write") logger.debug(message) self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_25_34, product_key=AceConfig.acekey): self._product_key = product_key self._gender = gender self._age = age self._seekback = AceConfig.videoseekback logger = logging.getLogger("AceClient_aceInit") self._write(AceMessage.request.HELLO) # Sending HELLOBG if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" # HELLOTS not resived from engine logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("AceInit ended") def _getResult(self): try: result = self._result.get(timeout=self._resulttimeout) if not result: raise AceException("Result not received from %s:%s" % (AceConfig.acehost, AceConfig.aceAPIport)) except gevent.Timeout: raise AceException("gevent_Timeout") return result def START(self, datatype, value, stream_type): ''' Start video method ''' if stream_type == 'hls': stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \ + ' transcode_mp3=' + str(AceConfig.transcode_mp3) \ + ' transcode_ac3=' + str(AceConfig.transcode_ac3) \ + ' preferred_audio_language=' + AceConfig.preferred_audio_language else: stream_type = 'output_format=http' self._urlresult = AsyncResult() self._write( AceMessage.request.START(datatype.upper(), value, stream_type)) self._getResult() def STOP(self): ''' Stop video method ''' if self._state is not None and self._state != '0': self._result = AsyncResult() self._write(AceMessage.request.STOP) self._getResult() def LOADASYNC(self, datatype, params): self._result = AsyncResult() self._write( AceMessage.request.LOADASYNC( datatype.upper(), random.randint(1, AceConfig.maxconns * 10000), params)) return self._getResult() def CONTENTINFO(self, datatype, value): dict = { 'torrent': 'url', 'infohash': 'infohash', 'raw': 'data', 'pid': 'content_id' } paramsdict = { dict[datatype]: value, 'developer_id': '0', 'affiliate_id': '0', 'zone_id': '0' } return self.LOADASYNC(datatype, paramsdict) def GETCID(self, datatype, url): contentinfo = self.CONTENTINFO(datatype, url) self._cidresult = AsyncResult() self._write( AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0)) cid = self._cidresult.get(True, 5) return '' if not cid or cid == '' else cid[2:] def GETINFOHASH(self, datatype, url): infohash = self.CONTENTINFO(datatype, url).get('infohash') return '' if not infohash or infohash == '' else infohash def getUrl(self, timeout=30): logger = logging.getLogger("AceClient_getURL") # Logger try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "Engine response time exceeded. GetURL timeout!" logger.error(errmsg) raise AceException(errmsg) def startStreamReader(self, url, cid, counter, req_headers=None): logger = logging.getLogger("StreamReader") logger.debug("Opening video stream: %s" % url) self._streamReaderState = 1 transcoder = None if 'range' in req_headers: del req_headers['range'] logger.debug("Get headers from client: %s" % req_headers) try: connection = self._streamReaderConnection = requests.get( url, headers=req_headers, stream=True) if connection.status_code not in (200, 206): logger.error("Failed to open video stream %s" % url) return None if url.endswith('.m3u8'): self._streamReaderConnection.headers = { 'Content-Type': 'application/octet-stream', 'Connection': 'Keep-Alive', 'Keep-Alive': 'timeout=15, max=100' } popen_params = { "bufsize": AceConfig.readchunksize, "stdout": PIPE, "stderr": None, "shell": False } if AceConfig.osplatform == 'Windows': ffmpeg_cmd = 'ffmpeg.exe ' CREATE_NO_WINDOW = 0x08000000 CREATE_NEW_PROCESS_GROUP = 0x00000200 DETACHED_PROCESS = 0x00000008 popen_params.update(creationflags=CREATE_NO_WINDOW | DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP) else: ffmpeg_cmd = 'ffmpeg ' ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url transcoder = psutil.Popen(ffmpeg_cmd.split(), **popen_params) out = transcoder.stdout logger.warning( "HLS stream detected. Ffmpeg transcoding started") else: out = connection.raw except requests.exceptions.RequestException: logger.error("Failed to open video stream") logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) else: with self._lock: self._streamReaderState = 2 self._lock.notifyAll() while True: data = None clients = counter.getClients(cid) if clients: try: data = out.read(AceConfig.readchunksize) except: logger.debug("No data received") pass else: with self._lock: if len(self._streamReaderQueue ) == AceConfig.readcachesize: self._streamReaderQueue.popleft() self._streamReaderQueue.append(data) for c in clients: try: c.addChunk(data, 5.0) except Queue.Full: if len(clients) > 1: logger.debug("Disconnecting client: %s" % c) c.destroy() else: logger.debug( "All clients disconnected - closing video stream") break finally: self.closeStreamReader() if transcoder: try: transcoder.kill() logger.warning("Ffmpeg transcoding stoped") except: pass with self._lock: self._streamReaderState = 3 self._lock.notifyAll() counter.deleteAll(cid) def closeStreamReader(self): logger = logging.getLogger("StreamReader") c = self._streamReaderConnection if c: c.close() self._streamReaderConnection = None logger.debug("Video stream closed") self._streamReaderQueue.clear() def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' return self._resumeevent.wait(timeout=timeout) def play(self): self._write(AceMessage.request.PLAY) def pause(self): self._write(AceMessage.request.PAUSE) def stop(self): self._write(AceMessage.request.STOP) def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n").strip() logger.debug('<<< ' + self._recvbuffer) except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return else: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'version_code=' in self._recvbuffer: v = self._recvbuffer.find('version_code=') self._engine_version_code = int( self._recvbuffer[v + 13:v + 20]) if 'key=' in self._recvbuffer: self._request_key_begin = self._recvbuffer.find('key=') self._request_key = self._recvbuffer[ self._request_key_begin + 4:self._request_key_begin + 14] self._write( AceMessage.request.READY_key( self._request_key, self._product_key)) self._request_key = None else: self._write(AceMessage.request.READY_nokey) # NOTREADY elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): logger.error("Ace engine is not ready. Wrong auth?") self._auth = None self._authevent.set() # LOADRESP elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): _contentinfo = json.loads(' '.join( self._recvbuffer.split()[2:])) if _contentinfo.get('status') == 100: logger.error( "LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: self._result.set(_contentinfo) #logger.debug("Content info: %s", _contentinfo) # START elif self._recvbuffer.startswith(AceMessage.response.START): if not self._seekback or self._started_again or not self._recvbuffer.endswith( ' stream=1'): # If seekback is disabled, we use link in first START command. # If seekback is enabled, we wait for first START command and # ignore it, then do seekback in first EVENT position command # AceStream sends us STOP and START again with new link. # We use only second link then. try: self._urlresult.set(self._recvbuffer.split()[1]) self._resumeevent.set() except IndexError as e: self._url = None else: logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS) # STOP elif self._recvbuffer.startswith(AceMessage.response.STOP): pass # SHUTDOWN elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return # AUTH elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA( self._gender, self._age)) except: pass self._authevent.set() # GETUSERDATA elif self._recvbuffer.startswith( AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") # LIVEPOS elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS): self._position = self._recvbuffer.split() self._position_last = self._position[2].split('=')[1] self._position_buf = self._position[9].split('=')[1] self._position = self._position[4].split('=')[1] if self._seekback and not self._started_again: self._write( AceMessage.request.LIVESEEK( str(int(self._position_last) - self._seekback))) logger.debug('Seeking back') self._started_again = True # DOWNLOADSTOP elif self._recvbuffer.startswith( AceMessage.response.DOWNLOADSTOP): self._state = self._recvbuffer.split()[1] # STATE elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] # INFO elif self._recvbuffer.startswith(AceMessage.response.INFO): self._state = self._recvbuffer.split()[1] # STATUS elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split( ';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to %s" % self._status) if self._status == 'main:err': logger.error(self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._status == 'main:idle': self._result.set(True) # PAUSE elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() # RESUME elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") self._resumeevent.set() # CID elif self._recvbuffer.startswith('##') or len( self._recvbuffer) == 0: self._cidresult.set(self._recvbuffer) logger.debug("CID: %s" % self._recvbuffer)
class AceClient: def __init__(self, host, port, connect_timeout = 5, result_timeout = 5, debug = logging.ERROR): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Debug level self._debug = debug # Current STATUS self._status = None # Current STATE self._state = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Logger logger = logging.getLogger('AceClient_init') try: self._socket = telnetlib.Telnet(host, port, connect_timeout) logger.info("Successfully connected with Ace!") except Exception as e: raise AceException("Socket creation error! Ace is not running? " + str(e)) # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def _write(self, message): try: self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + str(e)) def aceInit(self, gender = AceConst.SEX_MALE, age = AceConst.AGE_18_24, product_key = None, pause_delay = 0): self._product_key = product_key self._gender = gender self._age = age # PAUSE/RESUME delay self._pausedelay = pause_delay # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def START(self, datatype, value): ''' Start video method ''' # Logger logger = logging.getLogger("AceClient_START") self._result = AsyncResult() self._urlresult = AsyncResult() self._write(AceMessage.request.START(datatype.upper(), value)) try: if not self._result.get(timeout = self._resulttimeout): errmsg = "START error!" logger.error(errmsg) raise AceException(errmsg) except gevent.Timeout: errmsg = "START timeout!" logger.error(errmsg) raise AceException(errmsg) def getUrl(self, timeout = 40): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout = timeout) return res except gevent.Timeout: errmsg = "getURL timeout!" logger.error(errmsg) raise AceException(errmsg) def getPlayEvent(self, timeout = None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' self._resumeevent.wait(timeout = timeout) return def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n", 1) self._recvbuffer = self._recvbuffer.strip() except: # If something happened during read, abandon reader # Should not ever happen logger.error("Exception at socket read") return # Parsing everything if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'key=' in self._recvbuffer: self._request_key = self._recvbuffer.split()[2].split('=')[1] self._write(AceMessage.request.READY_key(self._request_key, self._product_key)) self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") return elif self._recvbuffer.startswith(AceMessage.response.START): # START try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None elif self._recvbuffer.startswith(AceMessage.response.STOP): pass elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write(AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to "+self._status) if self._status == 'main:err': logger.warning(self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception(AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception(AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep(self._pausedelay) self._resumeevent.set()
class AceClient(object): def __init__(self, host, port, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Logger logger = logging.getLogger('AceClient_init') try: self._socket = telnetlib.Telnet(host, port, connect_timeout) logger.info("Successfully connected with Ace!") except Exception as e: raise AceException( "Socket creation error! Ace is not running? " + repr(e)) # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def _write(self, message): try: self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0): self._product_key = product_key self._gender = gender self._age = age # PAUSE/RESUME delay self._pausedelay = pause_delay # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def _getResult(self): # Logger logger = logging.getLogger("AceClient_START") try: result = self._result.get(timeout=self._resulttimeout) if not result: errmsg = "START error!" logger.error(errmsg) raise AceException(errmsg) except gevent.Timeout: errmsg = "START timeout!" logger.error(errmsg) raise AceException(errmsg) return result def START(self, datatype, value): ''' Start video method ''' self._result = AsyncResult() self._urlresult = AsyncResult() self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value)) contentinfo = self._getResult() self._write(AceMessage.request.START(datatype.upper(), value)) self._getResult() return contentinfo def getUrl(self, timeout=40): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "getURL timeout!" logger.error(errmsg) raise AceException(errmsg) def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' self._resumeevent.wait(timeout=timeout) return def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n") self._recvbuffer = self._recvbuffer.strip() except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return if self._recvbuffer: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'key=' in self._recvbuffer: self._request_key = self._recvbuffer.split()[ 2].split('=')[1] try: self._write(AceMessage.request.READY_key( self._request_key, self._product_key, self._resulttimeout)) except urllib2.URLError as e: logger.error("Can't connect to keygen server! " + \ repr(e)) self._auth = False self._authevent.set() self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") self._auth = False self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): # LOADRESP _contentinfo_raw = self._recvbuffer.split()[2:] _contentinfo_raw = ' '.join(_contentinfo_raw) _contentinfo = json.loads(_contentinfo_raw) if _contentinfo.get('status') == 100: logger.error("LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: logger.debug("Content info: %s", _contentinfo) _filename = urllib2.unquote(_contentinfo.get('files')[0][0]) self._result.set(_filename) elif self._recvbuffer.startswith(AceMessage.response.START): # START try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None elif self._recvbuffer.startswith(AceMessage.response.STOP): pass elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to " + self._status) if self._status == 'main:err': logger.error( self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep(self._pausedelay) self._resumeevent.set()
class PriorityQueue(object): """A priority queue. It is greenlet-safe, and offers the ability of changing priorities and removing arbitrary items. The queue is implemented as a custom min-heap. The priority is a mix of a discrete priority level and of the timestamp. The elements of the queue are QueueItems. """ PRIORITY_EXTRA_HIGH = 0 PRIORITY_HIGH = 1 PRIORITY_MEDIUM = 2 PRIORITY_LOW = 3 PRIORITY_EXTRA_LOW = 4 def __init__(self): """Create a priority queue.""" # The queue: a min-heap whose elements are of the form # (priority, timestamp, item), where item is the actual data. self._queue = [] # Reverse lookup for the items in the queue: a dictionary # associating the index in the queue to each item. self._reverse = {} # Event to signal that there are items in the queue. self._event = Event() # Index of the next element that will be added to the queue. self._next_index = 0 def _verify(self): """Make sure that the internal state of the queue is consistent. This is used only for testing. """ if len(self._queue) != len(self._reverse): return False if len(self._queue) != self.length(): return False if self.empty() != (self.length() == 0): return False if self._event.isSet() == self.empty(): return False for item, idx in self._reverse.iteritems(): if self._queue[idx].item != item: return False return True def __contains__(self, item): """Implement the 'in' operator for an item in the queue. item (QueueItem): an item to search. return (bool): True if item is in the queue. """ return item in self._reverse def _swap(self, idx1, idx2): """Swap two elements in the queue, keeping their reverse indices up to date. idx1 (int): the index of the first element. idx2 (int): the index of the second element. """ self._queue[idx1], self._queue[idx2] = \ self._queue[idx2], self._queue[idx1] self._reverse[self._queue[idx1].item] = idx1 self._reverse[self._queue[idx2].item] = idx2 def _up_heap(self, idx): """Take the element in position idx up in the heap until its position is the right one. idx (int): the index of the element to lift. return (int): the new index of the element. """ while idx > 0: parent = (idx - 1) // 2 if self._queue[idx] < self._queue[parent]: self._swap(parent, idx) idx = parent else: break return idx def _down_heap(self, idx): """Take the element in position idx down in the heap until its position is the right one. idx (int): the index of the element to lower. return (int): the new index of the element. """ last = len(self._queue) - 1 while 2 * idx + 1 <= last: child = 2 * idx + 1 if 2 * idx + 2 <= last and \ self._queue[2 * idx + 2] < self._queue[child]: child = 2 * idx + 2 if self._queue[child] < self._queue[idx]: self._swap(child, idx) idx = child else: break return idx def _updown_heap(self, idx): """Perform both operations of up_heap and down_heap on an element. idx (int): the index of the element to lift. return (int): the new index of the element. """ idx = self._up_heap(idx) return self._down_heap(idx) def push(self, item, priority=None, timestamp=None): """Push an item in the queue. If timestamp is not specified, uses the current time. item (QueueItem): the item to add to the queue. priority (int|None): the priority of the item, or None for medium priority. timestamp (datetime|None): the time of the submission, or None to use now. return (bool): false if the element was already in the queue and was not pushed again, true otherwise.. """ if item in self._reverse: return False if priority is None: priority = PriorityQueue.PRIORITY_MEDIUM if timestamp is None: timestamp = make_datetime() index = self._next_index self._next_index += 1 self._queue.append(QueueEntry(item, priority, timestamp, index)) last = len(self._queue) - 1 self._reverse[item] = last self._up_heap(last) # Signal to listener greenlets that there might be something. self._event.set() return True def top(self, wait=False): """Return the first element in the queue without extracting it. wait (bool): if True, block until an element is present. return (QueueEntry): first element in the queue. raise (LookupError): on empty queue if wait was false. """ if not self.empty(): return self._queue[0] else: if not wait: raise LookupError("Empty queue.") else: while True: if self.empty(): self._event.wait() continue return self._queue[0] def pop(self, wait=False): """Extract (and return) the first element in the queue. wait (bool): if True, block until an element is present. return (QueueEntry): first element in the queue. raise (LookupError): on empty queue, if wait was false. """ top = self.top(wait) last = len(self._queue) - 1 self._swap(0, last) del self._reverse[top.item] del self._queue[last] # last is 0 when the queue becomes empty. if last > 0: self._down_heap(0) else: # Signal that there is nothing left for listeners. self._event.clear() return top def remove(self, item): """Remove an item from the queue. Raise a KeyError if not present. item (QueueItem): the item to remove. return (QueueEntry): the complete entry removed. raise (KeyError): if item not present. """ pos = self._reverse[item] entry = self._queue[pos] last = len(self._queue) - 1 self._swap(pos, last) del self._reverse[item] del self._queue[last] if pos != last: self._updown_heap(pos) if self.empty(): self._event.clear() return entry def set_priority(self, item, priority): """Change the priority of an item inside the queue. Raises an exception if the item is not in the queue. item (QueueItem): the item whose priority needs to change. priority (int): the new priority. raise (LookupError): if item not present. """ pos = self._reverse[item] self._queue[pos].priority = priority self._updown_heap(pos) def length(self): """Return the number of elements in the queue. return (int): length of the queue """ return len(self._queue) def empty(self): """Return if the queue is empty. return (bool): is the queue empty? """ return self.length() == 0 def get_status(self): """Return the content of the queue. Note that the order may be not correct, but the first element is the one at the top. return ([QueueEntry]): a list of entries containing the representation of the item, the priority and the timestamp. """ return [{ 'item': entry.item.to_dict(), 'priority': entry.priority, 'timestamp': make_timestamp(entry.timestamp) } for entry in self._queue]
class AceClient: def __init__(self, host, port, connect_timeout = 5, debug = logging.ERROR): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Debug level self._debug = debug # Current STATUS self._status = None # Current STATE self._state = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() self._resumeevent = Event() # Logging init logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s: %(message)s', datefmt='%d.%m.%Y %H:%M:%S', level=self._debug) logger = logging.getLogger('AceClient_init') try: self._socket = telnetlib.Telnet(host, port, connect_timeout) logger.debug("Successfully connected with Ace!") except Exception as e: raise AceException("Socket creation error! Ace is not running? " + str(e)) gevent.spawn(self._recvData) gevent.sleep() def __del__(self): self.destroy() def destroy(self): logger = logging.getLogger("ace_destroy") self._resumeevent.set() self._urlresult.set() if self._shuttingDown.isSet(): # Already in the middle of destroying return if self._socket: try: logger.debug("Destroying client...") self._write(AceMessage.request.SHUTDOWN) self._shuttingDown.set() except: # Ignore exceptions on destroy pass def _write(self, message): try: self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + str(e)) def aceInit(self, gender = AceConst.SEX_MALE, age = AceConst.GENDER_18_24, product_key = None, pause_delay = 0): self._product_key = product_key self._gender = gender self._age = age self._pausedelay = pause_delay self._write(AceMessage.request.HELLO) if not self._authevent.wait(5): logging.error("aceInit event timeout. Wrong key?") return if not self._auth: logging.error("aceInit auth error. Wrong key?") return logging.debug("aceInit ended") def START(self, datatype, value): self._result = AsyncResult() self._urlresult = AsyncResult() if datatype.lower() == 'pid': self._write(AceMessage.request.START('PID', {'content_id': value})) elif datatype.lower() == 'torrent': self._write(AceMessage.request.START('TORRENT', {'url': value})) if not self._result.get(): raise AceException("START error!") return def getUrl(self): res = self._urlresult.get() if res: return res else: return False def getPlayEvent(self): self._resumeevent.wait() return def _recvData(self): logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() if self._shuttingDown.isSet(): logger.debug("Shutting down is in the process, returning from _recvData...") self._socket.close() return try: self._recvbuffer = self._socket.read_until("\r\n", 1) except Exception as e: if self._shuttingDown.isSet(): logger.debug("Shutting down is in the process, returning from _recvData after socket_read...") else: raise e # Parsing everything if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'key=' in self._recvbuffer: self._request_key = self._recvbuffer.split()[2].split('=')[1] self._write(AceMessage.request.READY_key(self._request_key, self._product_key)) self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY # Not implemented yet logger.error("Ace is not ready. Wrong auth?") pass elif self._recvbuffer.startswith(AceMessage.response.START): # START try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None elif self._recvbuffer.startswith(AceMessage.response.STOP): pass elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): self.destroy() return elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write(AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to "+self._status) if self._status == 'main:err': logger.warning(self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set(False) self._urlresult.set(False) if self._status == 'main:starting': self._result.set(True) elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep(self._pausedelay) self._resumeevent.set()