def test_put_nowait_simple(self):
result = []
q = queue.Queue(1)
def store_result(func, *args):
result.append(func(*args))
core.active_event(store_result, util.wrap_errors(Exception, q.put_nowait), 2)
core.active_event(store_result, util.wrap_errors(Exception, q.put_nowait), 3)
gevent.sleep(0)
assert len(result) == 2, result
assert result[0] == None, result
assert isinstance(result[1], queue.Full), result
def test_put_nowait_simple(self):
result = []
q = queue.Queue(1)
def store_result(func, *args):
result.append(func(*args))
run_callback = get_hub().loop.run_callback
run_callback(store_result, util.wrap_errors(Full, q.put_nowait), 2)
run_callback(store_result, util.wrap_errors(Full, q.put_nowait), 3)
gevent.sleep(0)
assert len(result) == 2, result
assert result[0] is None, result
assert isinstance(result[1], queue.Full), result
def test_put_nowait_simple(self):
result = []
q = queue.Queue(1)
def store_result(func, *args):
result.append(func(*args))
core.active_event(store_result,
util.wrap_errors(Exception, q.put_nowait), 2)
core.active_event(store_result,
util.wrap_errors(Exception, q.put_nowait), 3)
gevent.sleep(0)
assert len(result) == 2, result
assert result[0] == None, result
assert isinstance(result[1], queue.Full), result
def test_put_nowait_simple(self):
result = []
q = queue.Queue(1)
def store_result(func, *args):
result.append(func(*args))
run_callback = get_hub().loop.run_callback
run_callback(store_result, util.wrap_errors(Full, q.put_nowait), 2)
run_callback(store_result, util.wrap_errors(Full, q.put_nowait), 3)
gevent.sleep(0)
assert len(result) == 2, result
assert result[0] is None, result
assert isinstance(result[1], queue.Full), result
def get_hostnames(ips, timeout):
"""
Do DNS resolution for a given list of IPs
Args:
ips (list): A list of IPs
timeout (int): The number of seconds to wait for resolution of **all** IPs
Returns:
list: A list of (address, hosname) tuples in the same order as the input list of IPs
"""
assert validators.PanoptesValidators.valid_nonempty_iterable_of_strings(
ips), u'ips should be a list'
assert validators.PanoptesValidators.valid_nonzero_integer(
timeout), u'timeout should be an int greater than zero'
jobs = [
gevent.spawn(wrap_errors((gaierror, herror), socket.gethostbyaddr), ip)
for ip in ips
]
gevent.joinall(jobs, timeout=timeout)
hostnames = [
None if isinstance(job.get(), (gaierror, herror)) else job.value
for job in jobs
]
results = {
ips[i]: unknown_hostname(ips[i]) if
((not result) or
(not result[0]) or result[0].startswith(u'UNKNOWN')) else result[0]
for i, result in enumerate(hostnames)
}
return results
def test_get_nowait_simple(self):
result = []
q = queue.Queue(1)
q.put(4)
def store_result(func, *args):
result.append(func(*args))
run_callback = get_hub().loop.run_callback
run_callback(store_result, util.wrap_errors(Empty, q.get_nowait))
run_callback(store_result, util.wrap_errors(Empty, q.get_nowait))
gevent.sleep(0)
assert len(result) == 2, result
assert result[0] == 4, result
assert isinstance(result[1], queue.Empty), result
def test_get_nowait_simple(self):
result = []
q = queue.Queue(1)
q.put(4)
def store_result(func, *args):
result.append(func(*args))
run_callback = get_hub().loop.run_callback
run_callback(store_result, util.wrap_errors(Empty, q.get_nowait))
run_callback(store_result, util.wrap_errors(Empty, q.get_nowait))
gevent.sleep(0)
assert len(result) == 2, result
assert result[0] == 4, result
assert isinstance(result[1], queue.Empty), result
def test_sleep_invalid_switch(self):
p = gevent.spawn(util.wrap_errors(AssertionError, gevent.sleep), 2)
switcher = gevent.spawn(p.switch, None)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
switcher.kill()
def test_wait_write_invalid_switch(self):
p = gevent.spawn(util.wrap_errors(AssertionError, socket.wait_write), 0)
switcher = gevent.spawn(p.switch, None)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
switcher.kill()
def test_wait_read_invalid_switch(self):
p = gevent.spawn(util.wrap_errors(AssertionError, socket.wait_read), 0)
switcher = gevent.spawn(p.switch, None)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
switcher.kill()
def resolve_hostnames(hostnames, timeout):
"""
Do DNS resolution for a given list of hostnames
This function uses gevent to resolve all the hostnames in *parallel*
Args:
hostnames (list): A list of strings
timeout (int): The number of seconds to wait for resolution of **all** hostnames
Returns:
list: A list of (hostname, address) tuples in the same order as the input list of hostnames
"""
assert validators.PanoptesValidators.valid_nonempty_iterable_of_strings(
hostnames), u'hostnames should be a list'
assert validators.PanoptesValidators.valid_nonzero_integer(
timeout), u'timeout should be an int greater than zero'
jobs = [
gevent.spawn(wrap_errors(gaierror, socket.gethostbyname), host)
for host in hostnames
]
gevent.joinall(jobs, timeout=timeout)
addresses = [
job.value if not isinstance(job.get(), gaierror) else None
for job in jobs
]
results = [(hostnames[i], result) for i, result in enumerate(addresses)]
return results
def test_sleep_invalid_switch(self):
p = gevent.spawn(util.wrap_errors(AssertionError, gevent.sleep), 2)
switcher = gevent.spawn(p.switch, None)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
switcher.kill()
def test_sleep_invalid_switch(self):
p = gevent.spawn(util.wrap_errors(AssertionError, gevent.sleep), 2)
gevent.sleep(0) # wait for p to start, because actual order of switching is reversed
switcher = gevent.spawn(p.switch, None)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
switcher.kill()
def test_sleep_invalid_switch(self):
p = gevent.spawn(util.wrap_errors(AssertionError, gevent.sleep), 2)
gevent.sleep(0) # wait for p to start, because actual order of switching is reversed
switcher = gevent.spawn(p.switch, None)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
switcher.kill()
def setup(self):
TelnetHandlerBase.setup(self)
# Spawn a greenlet to handle socket input
self.greenlet_ic = gevent.spawn(wrap_errors((socket.error), self.inputcooker))
# Note that inputcooker exits on EOF
# Sleep for 0.5 second to allow options negotiation
gevent.sleep(0.5)
def setup(self):
TelnetHandlerBase.setup(self)
# Spawn a greenlet to handle socket input
self.greenlet_ic = gevent.spawn(
wrap_errors((socket.error), self.inputcooker))
# Note that inputcooker exits on EOF
# Sleep for 0.5 second to allow options negotiation
gevent.sleep(0.5)
def test_wait_write_invalid_switch(self):
sock = socket.socket()
p = gevent.spawn(util.wrap_errors(AssertionError, socket.wait_write), sock.fileno())
gevent.sleep(0)
switcher = gevent.spawn(p.switch, None)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
switcher.kill()
def test_wait_read_invalid_switch(self):
sock = socket.socket()
p = gevent.spawn(util.wrap_errors(AssertionError, socket.wait_read), sock.fileno())
gevent.sleep(0)
switcher = gevent.spawn(p.switch, None)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
switcher.kill()
def get(self, uid=None):
next_url = request.args.get('next_url', None)
if next_url and 'instagram' not in next_url:
next_url = signer.loads(next_url)
api = InstagramAPI(access_token=request.access_token)
user = gevent.spawn(wrap_errors(InstagramAPIError, api.user),
user_id=uid)
feeds = gevent.spawn(wrap_errors(InstagramAPIError,
api.user_recent_media),
user_id=uid,
with_next_url=next_url)
if request.uid:
isfollows = spawn(isfollow, uid, api)
else:
isfollows = spawn(lambda x: False, uid)
gevent.joinall([user, feeds, isfollows])
user, feeds, isfollows = user.get(), feeds.get(), isfollows.get()
errors = [
e for e in (user, feeds, isfollows)
if isinstance(e, InstagramAPIError)
]
if errors:
if any([e.error_type == 'APINotAllowedError' for e in errors]):
return render_template('profile-noauth.html', uid=uid)
if any([e.error_type == 'APINotFoundError' for e in errors]):
return NotFound(u'User Not Found')
return InternalServerError('Internal Server Error')
next_url = feeds[1] if feeds else None
next_url = signer.dumps(next_url) if next_url else next_url
feeds = feeds[0] if feeds else []
isme = False
if request.uid and uid == request.uid:
isme = True
return render_template('profile.html',
user=user,
feeds=feeds,
isme=isme,
isfollow=isfollows,
next_url=next_url)
def get(self, ukey):
next_url = request.args.get('next_url', None)
if next_url and 'instagram' not in next_url:
next_url = signer.loads(next_url)
api = InstagramAPI(access_token=request.access_token)
user = gevent.spawn(wrap_errors(InstagramAPIError, api.user),
user_id=ukey)
feeds = gevent.spawn(wrap_errors(InstagramAPIError,
api.user_recent_media),
user_id=ukey, with_next_url=next_url)
if request.ukey:
isfollows = spawn(isfollow, ukey, api)
else:
isfollows = spawn(lambda x: False, ukey)
gevent.joinall([user, feeds, isfollows])
user, feeds, isfollows = user.get(), feeds.get(), isfollows.get()
errors = [e for e in (user, feeds, isfollows)
if isinstance(e, InstagramAPIError)]
if errors:
if any([e.error_type == 'APINotAllowedError' for e in errors]):
return render('profile-noauth.html', ukey=ukey)
if any([e.error_type == 'APINotFoundError' for e in errors]):
return notfound(u'用户不存在')
app.logger.error([str(e) for e in errors])
return apierror(u'服务器暂时出问题了')
next_url = feeds[1] if feeds else None
next_url = signer.dumps(next_url) if next_url else next_url
feeds = feeds[0] if feeds else []
isme = False
if request.ukey and ukey == request.ukey:
isme = True
return render(
'profile.html',
user=user,
feeds=feeds,
isme=isme,
isfollow=isfollows,
next_url=next_url
)
def get(self, uid=None):
next_url = request.args.get('next_url', None)
if next_url and 'instagram' not in next_url:
next_url = signer.loads(next_url)
api = InstagramAPI(access_token=request.access_token)
user = gevent.spawn(wrap_errors(InstagramAPIError, api.user),
user_id=uid)
feeds = gevent.spawn(wrap_errors(InstagramAPIError,
api.user_recent_media),
user_id=uid, with_next_url=next_url)
if request.uid:
isfollows = spawn(isfollow, uid, api)
else:
isfollows = spawn(lambda x: False, uid)
gevent.joinall([user, feeds, isfollows])
user, feeds, isfollows = user.get(), feeds.get(), isfollows.get()
errors = [e for e in (user, feeds, isfollows)
if isinstance(e, InstagramAPIError)]
if errors:
if any([e.error_type == 'APINotAllowedError' for e in errors]):
return render_template('profile-noauth.html', uid=uid)
if any([e.error_type == 'APINotFoundError' for e in errors]):
return NotFound(u'User Not Found')
return InternalServerError('Internal Server Error')
next_url = feeds[1] if feeds else None
next_url = signer.dumps(next_url) if next_url else next_url
feeds = feeds[0] if feeds else []
isme = False
if request.uid and uid == request.uid:
isme = True
return render_template(
'profile.html',
user=user,
feeds=feeds,
isme=isme,
isfollow=isfollows,
next_url=next_url)
def _test_wait_read_invalid_switch(self, sleep):
sock1, sock2 = socket.socketpair()
try:
p = gevent.spawn(util.wrap_errors(AssertionError, socket.wait_read), sock1.fileno())
gevent.get_hub().loop.run_callback(switch_None, p)
if sleep is not None:
gevent.sleep(sleep)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
finally:
sock1.close()
sock2.close()
def spawn_greenlet(func, *args, **kwargs):
t = gevent.spawn(wrap_errors(Exception, func), *args)
t._get = t.get
def new_get(self, *args, **kwargs):
ret = self._get(*args, **kwargs)
if isinstance(ret, Exception):
raise ret
else:
return ret
setattr(t, "get", types.MethodType(new_get, t))
return t
def spawn_greenlet(func, *args, **kwargs):
t = gevent.spawn(wrap_errors(Exception, func), *args)
t._get = t.get
def new_get(self, *args, **kwargs):
ret = self._get(*args, **kwargs)
if isinstance(ret, Exception):
raise ret
else:
return ret
setattr(t, "get", types.MethodType(new_get, t))
return t
def _test_wait_read_invalid_switch(self, sleep):
sock1, sock2 = socket.socketpair()
try:
p = gevent.spawn(util.wrap_errors(AssertionError, socket.wait_read), sock1.fileno())
gevent.get_hub().loop.run_callback(switch_None, p)
if sleep is not None:
gevent.sleep(sleep)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
finally:
sock1.close()
sock2.close()
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_25_34,
product_key=None,
videoseekback=0,
videotimeout=30):
self._gender = gender
self._age = age
self._product_key = product_key
self._seekback = videoseekback
self._videotimeout = videotimeout
self._started_again.clear()
# Spawning telnet data reader with recvbuffer read timeout (allowable STATE 0 (IDLE) time)
gevent.spawn(
wrap_errors((EOFError, gevent.socket.error), self._recvData),
self._videotimeout).link_exception(lambda x: logging.error(
'Error reading data from AceEngine API port'))
self._auth = AsyncResult()
self._write(AceMessage.request.HELLO) # Sending HELLOBG
try:
params = self._auth.get(timeout=self._resulttimeout)
except gevent.Timeout as t:
errmsg = 'Engine response time %s exceeded. HELLOTS not resived!' % t
raise AceException(errmsg)
if isinstance(params, dict):
self._write(
AceMessage.request.READY(params.get('key', ''),
self._product_key))
else:
self._auth.set(params)
try:
if self._auth.get(
timeout=self._resulttimeout
) == 'NOTREADY': # Get NOTREADY instead AUTH user_auth_level
errmsg = 'NOTREADY recived from AceEngine! Wrong acekey?'
raise AceException(errmsg)
except gevent.Timeout as t:
errmsg = 'Engine response time %s exceeded. AUTH not resived!' % t
raise AceException(errmsg)
if int(params.get('version_code',
0)) >= 3003600: # Display download_stopped massage
params_dict = {'use_stop_notifications': '1'}
self._write(AceMessage.request.SETOPTIONS(params_dict))
def _cb_request(self, request):
try:
spawn = self.spawn
request.connection.set_closecb(self)
self._requests.setdefault(request.connection._obj, []).append(request)
if spawn is None:
self.handle(request)
else:
greenlet = spawn(wrap_errors(core.HttpRequestDeleted, self.handle), request)
rawlink = getattr(greenlet, 'rawlink', None)
if rawlink is not None:
greenlet._request = request
rawlink(self._cb_request_processed)
except:
traceback.print_exc()
try:
sys.stderr.write('Failed to handle request: %s\n\n' % (request, ))
except:
pass
self.reply_error(request)
def test_put_nowait_unlock(self):
result = []
q = queue.Queue(0)
p = gevent.spawn(q.get)
def store_result(func, *args):
result.append(func(*args))
assert q.empty(), q
assert q.full(), q
gevent.sleep(0)
assert q.empty(), q
assert q.full(), q
core.active_event(store_result, util.wrap_errors(Exception, q.put_nowait), 10)
assert not p.ready(), p
gevent.sleep(0)
assert result == [None], result
assert p.ready(), p
assert q.full(), q
assert q.empty(), q
def test_put_nowait_unlock(self):
result = []
q = queue.Queue(0)
p = gevent.spawn(q.get)
def store_result(func, *args):
result.append(func(*args))
assert q.empty(), q
assert q.full(), q
gevent.sleep(0)
assert q.empty(), q
assert q.full(), q
core.active_event(store_result,
util.wrap_errors(Exception, q.put_nowait), 10)
assert not p.ready(), p
gevent.sleep(0)
assert result == [None], result
assert p.ready(), p
assert q.full(), q
assert q.empty(), q
result = []
if family == AF_INET:
for res in resolve_ipv4(host, evdns_flags)[1]:
sockaddr = (inet_ntop(family, res), port)
for socktype, proto in socktype_proto:
result.append((family, socktype, proto, '', sockaddr))
elif family == AF_INET6:
for res in resolve_ipv6(host, evdns_flags)[1]:
sockaddr = (inet_ntop(family, res), port, 0, 0)
for socktype, proto in socktype_proto:
result.append((family, socktype, proto, '', sockaddr))
else:
failure = None
job = spawn(wrap_errors(gaierror, resolve_ipv6), host, evdns_flags)
try:
try:
ipv4_res = resolve_ipv4(host, evdns_flags)[1]
except gaierror, failure:
ipv4_res = None
ipv6_res = job.get()
if isinstance(ipv6_res, gaierror):
ipv6_res = None
if failure is not None:
raise
if ipv4_res is not None:
for res in ipv4_res:
sockaddr = (inet_ntop(AF_INET, res), port)
for socktype, proto in socktype_proto:
result.append((AF_INET, socktype, proto, '', sockaddr))
def spawn(fn, *args, **kwargs):
return gevent.spawn(wrap_errors(InstagramAPIError, fn), *args, **kwargs)
result = []
if family == AF_INET:
for res in resolve_ipv4(host, evdns_flags)[1]:
sockaddr = (inet_ntop(family, res), port)
for socktype, proto in socktype_proto:
result.append((family, socktype, proto, '', sockaddr))
elif family == AF_INET6:
for res in resolve_ipv6(host, evdns_flags)[1]:
sockaddr = (inet_ntop(family, res), port, 0, 0)
for socktype, proto in socktype_proto:
result.append((family, socktype, proto, '', sockaddr))
else:
failure = None
job = spawn(wrap_errors(gaierror, resolve_ipv6), host, evdns_flags)
try:
try:
ipv4_res = resolve_ipv4(host, evdns_flags)[1]
except gaierror, failure:
ipv4_res = None
ipv6_res = job.get()
if isinstance(ipv6_res, gaierror):
ipv6_res = None
if failure is not None:
raise
if ipv4_res is not None:
for res in ipv4_res:
sockaddr = (inet_ntop(AF_INET, res), port)
for socktype, proto in socktype_proto:
result.append(
def _start_watching(self, sock):
disconnected = lambda x: self.disconnected(sock)
recv = wrap_errors(socket.error, sock.recv)
gevent.spawn(recv, 1).link(disconnected)
def handleRequest(self,
headers_only,
channelName=None,
channelIcon=None,
fmt=None):
logger = logging.getLogger('HandleRequest')
self.reqparams, self.path = parse_qs(
self.query), self.path[:-1] if self.path.endswith(
'/') else self.path
videoextdefaults = ('.avi', '.flv', '.m2ts', '.mkv', '.mpeg', '.mpeg4',
'.mpegts', '.mpg4', '.mp4', '.mpg', '.mov', '.mpv',
'.qt', '.ts', '.wmv')
# Limit on the number of connected clients
if 0 < AceConfig.maxconns <= len(
AceProxy.clientcounter.getAllClientsList()):
self.dieWithError(
403,
"Maximum client connections reached, can't serve request from %s"
% self.clientip, logging.ERROR)
return
# Check if third parameter exists…/pid/blablablablabla/video.mpg
# |_________|
# And if it ends with regular video extension
try:
if not self.path.endswith(videoextdefaults):
self.dieWithError(
501,
'Request seems like valid but no valid video extension was provided',
logging.ERROR)
return
except IndexError:
self.dieWithError(400, 'Bad Request',
logging.WARNING) # 400 Bad Request
return
# Pretend to work fine with Fake or HEAD request.
if headers_only or AceConfig.isFakeRequest(self.path, self.query,
self.headers):
# Return 200 and exit
if headers_only:
logger.debug('Sending headers and closing connection')
else:
logger.debug('Fake request - closing connection')
self.send_response(200)
self.send_header('Content-Type', 'video/mp2t')
self.send_header('Connection', 'Close')
self.end_headers()
self.closeConnection()
return
# Make dict with parameters
# [file_indexes, developer_id, affiliate_id, zone_id, stream_id]
paramsdict = {}.fromkeys(aceclient.acemessages.AceConst.START_PARAMS,
'0')
for i in range(3, len(self.splittedpath)):
paramsdict[aceclient.acemessages.AceConst.START_PARAMS[
i - 3]] = self.splittedpath[i] if self.splittedpath[i].isdigit(
) else '0'
paramsdict[self.reqtype] = unquote(
self.splittedpath[2]) #self.path_unquoted
#End parameters dict
self.connectionTime = gevent.time.time()
self.sessionID = str(uuid4().int)[:8]
self.clientInfo = self.transcoder = None
self.channelIcon = 'http://static.acestream.net/sites/acestream/img/ACE-logo.png' if channelIcon is None else channelIcon
try:
if not AceProxy.clientcounter.idleAce:
logger.debug('Create connection to AceEngine.....')
AceProxy.clientcounter.idleAce = aceclient.AceClient(
AceProxy.clientcounter, AceConfig.ace,
AceConfig.aceconntimeout, AceConfig.aceresulttimeout)
AceProxy.clientcounter.idleAce.aceInit(AceConfig.acesex,
AceConfig.aceage,
AceConfig.acekey,
AceConfig.videoseekback,
AceConfig.videotimeout)
self.CID, self.channelName = AceProxy.clientcounter.idleAce.GETINFOHASH(
self.reqtype, paramsdict[self.reqtype], self.sessionID,
paramsdict['file_indexes'])
except aceclient.AceException as e:
AceProxy.clientcounter.idleAce = None
self.dieWithError(404, '%s' % repr(e), logging.ERROR)
return
mimetype = mimetypes.guess_type(self.channelName)[0]
try:
gevent.spawn(wrap_errors(
gevent.socket.error, self.rfile.read)).link(
lambda x: self.finish()) # Client disconection watchdog
self.q = gevent.queue.Queue(maxsize=AceConfig.videotimeout)
self.out = self.wfile
# If &fmt transcode key present in request
if fmt is None: fmt = self.reqparams.get('fmt', [''])[0]
if fmt and AceConfig.osplatform != 'Windows':
if fmt in AceConfig.transcodecmd:
stderr = None if AceConfig.loglevel == logging.DEBUG else DEVNULL
popen_params = {
'bufsize': 1048576,
'stdin': gevent.subprocess.PIPE,
'stdout': self.wfile,
'stderr': stderr,
'shell': False
}
try:
self.transcoder = gevent.event.AsyncResult()
AceProxy.spawn(lambda: psutil.Popen(
AceConfig.transcodecmd[fmt], **popen_params)).link(
self.transcoder)
self.transcoder = self.transcoder.get(timeout=2.0)
self.out = self.transcoder.stdin
logger.info('Transcoding for %s started' %
self.clientip)
except:
logger.error(
'Error starting transcoding! Is Ffmpeg or VLC installed?'
)
self.transcoder = None
self.out = self.wfile
else:
logger.error(
"Can't found fmt key. Transcoding not started!")
# Start broadcast if it does not exist
if AceProxy.clientcounter.addClient(self) == 1:
playback_url = self.ace.START(self.reqtype, paramsdict,
AceConfig.acestreamtype)
AceProxy.pool.spawn(
StreamReader, playback_url,
self.CID).link(lambda x: logging.debug(
'Broadcast "%s" stoped. Last client disconnected' %
self.channelName))
logger.info('Streaming "%s" to %s started' %
(self.channelName, self.clientip))
# Sending videostream headers to client
self.response_use_chunked = False if (
self.transcoder is not None or self.request_version
== 'HTTP/1.0') else AceConfig.use_chunked
drop_headers = []
proxy_headers = {
'Connection': 'keep-alive',
'Keep-Alive': 'timeout=15, max=100',
'Accept-Ranges': 'none',
'Transfer-Encoding': 'chunked',
'Content-Type': 'video/MP2T' if mimetype is None else mimetype,
'Pragma': 'no-cache',
'Cache-Control': 'max-age=0, no-cache, no-store'
}
if not self.response_use_chunked:
self.protocol_version = 'HTTP/1.0'
proxy_headers['Connection'] = 'Close'
drop_headers.extend(
['Transfer-Encoding', 'Keep-Alive', 'Cache-Control'])
response_headers = [(k, v) for (k, v) in proxy_headers.items()
if k not in drop_headers]
self.send_response(200)
logger.debug('Sending HTTPAceProxy headers to client: %s' %
dict(response_headers))
gevent.joinall([
gevent.spawn(self.send_header, k, v)
for (k, v) in response_headers
])
self.end_headers()
# write data to client while he is alive
for chunk in self.q:
self.out.write(chunk)
except aceclient.AceException as e:
AceProxy.pool.map(
lambda x: x.dieWithError(500, repr(e), logging.ERROR),
AceProxy.clientcounter.getClientsList(self.CID))
except (gevent.GreenletExit, gevent.socket.error):
pass # Client disconnected
finally:
AceProxy.clientcounter.deleteClient(self)
logging.info('Streaming "%s" to %s finished' %
(self.channelName, self.clientip))
if self.transcoder:
try:
self.transcoder.kill()
logging.info('Transcoding for %s stoped' % self.clientip)
except:
pass
self.closeConnection()
def spawn(fn, *args, **kwargs):
return gevent.spawn(wrap_errors(InstagramAPIError, fn), *args, **kwargs)
def handleRequest(self):
'''
Main request handler path: /{reqtype}/{reqtype_value}/{file_indexes}/{developer_id}/{affiliate_id}/{zone_id}/{stream_id}/{fname}.{ext}
'''
logger = logging.getLogger('handleRequest')
# Make handler parameters dict
self.__dict__.update({}.fromkeys(
aceclient.acemessages.AceConst.START_PARAMS, '0'
)) # [file_indexes, developer_id, affiliate_id, zone_id, stream_id]
self.__dict__.update({
k: v
for (k, v) in
[(aceclient.acemessages.AceConst.START_PARAMS[i - 3],
self.splittedpath[i] if self.splittedpath[i].isdigit() else '0')
for i in range(3, len(self.splittedpath))]
})
self.__dict__.update({
self.reqtype:
unquote(self.splittedpath[2]), # {reqtype: reqtype_value}
'ace':
AceConfig.ace,
'acesex':
AceConfig.acesex,
'aceage':
AceConfig.aceage,
'acekey':
AceConfig.acekey,
'connect_timeout':
AceConfig.aceconntimeout,
'result_timeout':
AceConfig.aceresulttimeout,
'videoseekback':
AceConfig.videoseekback,
'videotimeout':
AceConfig.videotimeout,
'stream_type':
' '.join([
'{}={}'.format(k, v)
for k, v in AceConfig.acestreamtype.items()
]), # request http or hls from AceEngine
'sessionID':
self.handlerGreenlet.name[
self.handlerGreenlet.name.rfind('-') +
1:], # Greenlet.minimal_ident A small, unique non-negative integer
'connectionTime':
gevent.time.time(),
'clientDetail':
None,
'channelIcon':
self.__dict__.get(
'channelIcon',
'http://static.acestream.net/sites/acestream/img/ACE-logo.png'
),
})
# End parameters dict
try:
self.q = gevent.queue.Queue(maxsize=AceConfig.videotimeout)
transcoder = gevent.event.AsyncResult()
out = self.wfile
gevent.spawn(wrap_errors(gevent.socket.error,
self.rfile.read)).link(
lambda x: self.handlerGreenlet.kill()
) # Client disconection watchdog
try:
if not AceProxy.clientcounter.idleAce:
logger.debug(
'Create a connection with AceStream on {ace[aceHostIP]}:{ace[aceAPIport]}'
.format(**self.__dict__))
AceProxy.clientcounter.idleAce = aceclient.AceClient(
self.__dict__)
AceProxy.clientcounter.idleAce.GetAUTH()
if self.reqtype not in ('direct_url', 'efile_url'):
self.__dict__.update(
AceProxy.clientcounter.idleAce.GetCONTENTINFO(
self.__dict__))
self.channelName = ensure_str(
self.__dict__.get(
'channelName',
next(
iter([
x[0] for x in self.files
if x[1] == int(self.file_indexes)
]), 'NoNameChannel')))
else:
self.channelName = ensure_str(
self.__dict__.get('channelName', 'NoNameChannel'))
self.infohash = requests.auth.hashlib.sha1(
ensure_binary(self.path)).hexdigest()
AceProxy.clientcounter.idleAce._title = self.channelName
except Exception as e:
AceProxy.clientcounter.idleAce._read.kill()
logging.debug('Error 404')
self.send_error(404, '%s' % repr(e), logging.ERROR)
self.ext = self.__dict__.get(
'ext', self.channelName[self.channelName.rfind('.') + 1:])
if self.ext == self.channelName:
self.ext = query_get(self.query, 'ext', 'ts')
mimetype = mimetypes.guess_type(
'{channelName}.{ext}'.format(**self.__dict__))[0]
try:
# If &fmt transcode key present in request
fmt = query_get(self.query, 'fmt')
if fmt:
if AceConfig.osplatform != 'Windows':
if fmt in AceConfig.transcodecmd:
stderr = None if AceConfig.loglevel == logging.DEBUG else DEVNULL
popen_params = {
'bufsize': 1048576,
'stdin': gevent.subprocess.PIPE,
'stdout': self.wfile,
'stderr': stderr,
'shell': False
}
try:
gevent.spawn(lambda: psutil.Popen(
AceConfig.transcodecmd[fmt], **popen_params
)).link(transcoder)
out = transcoder.get(timeout=2.0).stdin
logger.info(
'[{channelName}]: Transcoding to [{clientip}] started'
.format(**self.__dict__))
except:
logger.error(
'[{channelName}]: Error starting transcoding to [{clientip}]! Is ffmpeg or VLC installed?'
.format(**self.__dict__))
else:
logger.error(
"[{channelName}]: Can't found fmt key. Transcoding to [{clientip}] not started!"
.format(**self.__dict__))
elif AceConfig.osplatform == 'Windows':
logger.error(
'[{channelName}]: Not applicable in Windnows OS. Transcoding to [{clientip}] not started!'
.format(**self.__dict__))
if AceProxy.clientcounter.addClient(self) == 1:
# Create broadcast if it does not exist yet
gevent.spawn(
StreamReader,
self.ace.GetBroadcastStartParams(self.__dict__)
).link(lambda x: logger.debug(
'[{channelName}]: Broadcast destroyed. Last client disconnected'
.format(**self.__dict__)))
logger.debug('[{channelName}]: Broadcast created'.format(
**self.__dict__))
else:
logger.debug(
'[{channelName}]: Broadcast already exists'.format(
**self.__dict__))
logger.info(
'[{channelName}]: Streaming to [{clientip}] started'.
format(**self.__dict__))
# Sending videostream headers to client
response_use_chunked = False if (
transcoder.value or self.request_version
== 'HTTP/1.0') else AceConfig.use_chunked
drop_headers = []
proxy_headers = {
'Connection': 'keep-alive',
'Keep-Alive':
'timeout=%s, max=100' % AceConfig.videotimeout,
'Accept-Ranges': 'none',
'Transfer-Encoding': 'chunked',
'Content-Type':
'video/MP2T' if mimetype is None else mimetype,
}
if not response_use_chunked:
self.protocol_version = 'HTTP/1.0'
proxy_headers['Connection'] = 'Close'
drop_headers.extend(['Transfer-Encoding', 'Keep-Alive'])
response_headers = [(k, v) for (k, v) in proxy_headers.items()
if k not in drop_headers]
self.send_response(200)
logger.debug('[%s]: Sending HTTPAceProxy headers: %s' %
(self.clientip, dict(response_headers)))
gevent.joinall([
gevent.spawn(self.send_header, k, v)
for (k, v) in response_headers
])
self.end_headers()
# write data to client while it is alive
for chunk in self.q:
out.write(b'%X\r\n%s\r\n' %
(len(chunk),
chunk) if response_use_chunked else chunk)
except aceclient.AceException as e:
_ = AceProxy.pool.map(
lambda x: x.send_error(500, repr(e), logging.ERROR),
AceProxy.clientcounter.getClientsList(self.infohash))
except gevent.socket.error:
pass # Client disconnected
except gevent.GreenletExit:
AceProxy.clientcounter.idleAce._read.kill()
finally:
AceProxy.clientcounter.deleteClient(self)
logger.info(
'[{channelName}]: Streaming to [{clientip}] finished'.format(
**self.__dict__))
if transcoder.value:
try:
transcoder.value.kill()
logger.info(
'[{channelName}]: Transcoding to [{clientip}] stoped'.
format(**self.__dict__))
except:
pass
def handleRequest(self, **params):
'''
:params: dict() with keys: headers_only, channelName, channelIcon
'''
logger = logging.getLogger('HandleRequest')
self.path = self.path[:-1] if self.path.endswith('/') else self.path
# Limit on the number of connected clients
if 0 < AceConfig.maxconns <= len(
AceProxy.clientcounter.getAllClientsList()):
self.dieWithError(
403,
"Maximum client connections reached, can't serve request from %s"
% self.clientip, logging.ERROR)
return
# Check if third parameter exists…/pid/blablablablabla/video.mpg
# |_________|
# And if it ends with regular video extension
try:
if not self.path.endswith(
('.avi', '.flv', '.m2ts', '.mkv', '.mpeg', '.mpeg4', '.mpegts',
'.mpg4', '.mp4', '.mpg', '.mov', '.mpv', '.qt', '.ts',
'.wmv')):
self.dieWithError(
501,
'Request seems like valid but no valid video extension was provided',
logging.ERROR)
return
except IndexError:
self.dieWithError(400, 'Bad Request',
logging.WARNING) # 400 Bad Request
return
# Pretend to work fine with Fake or HEAD request.
if params.get('headers_only') or AceConfig.isFakeRequest(
self.path, self.query, self.headers):
# Return 200 and exit
if params.get('headers_only'):
logger.debug('Sending headers and closing connection')
else:
logger.debug('Fake request - closing connection')
self.send_response(200)
self.send_header('Content-Type', 'video/mp2t')
self.send_header('Connection', 'Close')
self.end_headers()
self.closeConnection()
return
# Make parameters dict
params.update({self.reqtype:
unquote(self.splittedpath[2])}) # {command: value}
params.update(
{}.fromkeys(aceclient.acemessages.AceConst.START_PARAMS, '0')
) # [file_indexes, developer_id, affiliate_id, zone_id, stream_id]
params.update({
k: v
for (k, v) in
[(aceclient.acemessages.AceConst.START_PARAMS[i - 3],
self.splittedpath[i] if self.splittedpath[i].isdigit() else '0')
for i in range(3, len(self.splittedpath))]
})
params.update({
'stream_type':
' '.join([
'{}={}'.format(k, v)
for k, v in AceConfig.acestreamtype.items()
])
}) # request http or hls from AceEngine
params['request_id'] = self.sessionID = str(uuid4().int)[:8]
# End parameters dict
self.connectionTime = gevent.time.time()
self.clientInfo = None
self.channelIcon = params.get('channelIcon')
if self.channelIcon is None:
self.channelIcon = 'http://static.acestream.net/sites/acestream/img/ACE-logo.png'
try:
if not AceProxy.clientcounter.idleAce:
logger.debug(
'Create connection with AceStream on {aceHostIP}:{aceAPIport}'
.format(**AceConfig.ace))
AceProxy.clientcounter.idleAce = aceclient.AceClient(
AceConfig.ace, AceConfig.aceconntimeout,
AceConfig.aceresulttimeout)
AceProxy.clientcounter.idleAce.GetAUTH(AceConfig.acesex,
AceConfig.aceage,
AceConfig.acekey,
AceConfig.videoseekback,
AceConfig.videotimeout)
if self.reqtype not in ('direct_url', 'efile_url'):
self.CID, self.channelName = AceProxy.clientcounter.idleAce.GetCONTENTINFO(
params)
else:
self.channelName = params.get('channelName')
if self.channelName is None: self.channelName = 'NoNameChannel'
self.CID = requests.auth.hashlib.sha1(
self.channelName.encode('utf-8')).hexdigest()
except aceclient.AceException as e:
AceProxy.clientcounter.idleAce = None
self.dieWithError(404, '%s' % repr(e), logging.ERROR)
return
ext = self.channelName[self.channelName.rfind('.') + 1:]
if ext == self.channelName:
ext = parse_qs(self.query).get('ext', ['ts'])[0]
mimetype = mimetypes.guess_type('%s.%s' % (self.channelName, ext))[0]
try:
AceProxy.pool.spawn(
wrap_errors(gevent.socket.error, self.rfile.read)).link(
lambda x: self.finish()) # Client disconection watchdog
self.q = gevent.queue.Queue(maxsize=AceConfig.videotimeout)
out = self.wfile
# If &fmt transcode key present in request
fmt = parse_qs(self.query).get('fmt', [''])[0]
transcoder = gevent.event.AsyncResult()
if fmt:
if AceConfig.osplatform != 'Windows':
if fmt in AceConfig.transcodecmd:
stderr = None if AceConfig.loglevel == logging.DEBUG else DEVNULL
popen_params = {
'bufsize': 1048576,
'stdin': gevent.subprocess.PIPE,
'stdout': self.wfile,
'stderr': stderr,
'shell': False
}
try:
#transcoder = gevent.event.AsyncResult()
AceProxy.pool.spawn(lambda: psutil.Popen(
AceConfig.transcodecmd[fmt], **popen_params)
).link(transcoder)
out = transcoder.get(timeout=2.0).stdin
logger.info(
'Transcoding for {clientip} started'.format(
**self.__dict__))
except:
logger.error(
'Error starting transcoding! Is Ffmpeg or VLC installed?'
)
else:
logger.error(
"Can't found fmt key. Transcoding not started!")
elif AceConfig.osplatform == 'Windows':
logger.error(
'Not applicable in Windnows OS. Transcoding not started!'
)
# Start broadcast if it does not exist
if AceProxy.clientcounter.addClient(self) == 1:
START = self.ace.GetBroadcastStartParams(params)
self.broadcast = AceProxy.pool.spawn(StreamReader,
START['url'],
START['infohash'])
self.broadcast.link(lambda x: logging.debug(
'Broadcast "{channelName}" stoped. Last client disconnected'
.format(**self.__dict__)))
logger.info(
'Streaming "{channelName}" to {clientip} started'.format(
**self.__dict__))
# Sending videostream headers to client
response_use_chunked = False if (
transcoder is not None or self.request_version
== 'HTTP/1.0') else AceConfig.use_chunked
drop_headers = []
proxy_headers = {
'Connection': 'keep-alive',
'Keep-Alive': 'timeout=%s, max=100' % AceConfig.videotimeout,
'Accept-Ranges': 'none',
'Transfer-Encoding': 'chunked',
'Content-Type': 'video/MP2T' if mimetype is None else mimetype
}
if not response_use_chunked:
self.protocol_version = 'HTTP/1.0'
proxy_headers['Connection'] = 'Close'
drop_headers.extend(['Transfer-Encoding', 'Keep-Alive'])
response_headers = [(k, v) for (k, v) in proxy_headers.items()
if k not in drop_headers]
self.send_response(200)
logger.debug('Sending HTTPAceProxy headers to client: %s' %
dict(response_headers))
gevent.joinall([
gevent.spawn(self.send_header, k, v)
for (k, v) in response_headers
])
self.end_headers()
# write data to client while he is alive
for chunk in self.q:
out.write(b'%x\r\n' % len(chunk) + chunk +
b'\r\n' if response_use_chunked else chunk)
except aceclient.AceException as e:
_ = AceProxy.pool.map(
lambda x: x.dieWithError(500, repr(e), logging.ERROR),
AceProxy.clientcounter.getClientsList(self.CID))
except (gevent.GreenletExit, gevent.socket.error):
pass # Client disconnected
finally:
AceProxy.clientcounter.deleteClient(self)
logging.info(
'Streaming "{channelName}" to {clientip} finished'.format(
**self.__dict__))
if transcoder.value:
try:
transcoder.value.kill()
logging.info('Transcoding for {clientip} stoped'.format(
**self.__dict__))
except:
pass
self.closeConnection()
return
def _start_watching(self, sock):
disconnected = lambda x: self.disconnected(sock)
recv = wrap_errors(socket.error, sock.recv)
gevent.spawn(recv, 1).link(disconnected)
start = time.time()
if 'data' in options and callable(options['data']):
options = copy(options)
options['data'] = options['data'](method, url, options)
if 'hook' in options:
method, url, options = options['hook'](method, url, options)
del options['hook']
res = method(url, **options)
_stats[res.status_code].append(time.time() - start)
sys.stdout.write('=')
sys.stdout.flush()
onecall = wrap_errors((RequestException, ), _onecall)
def run(url, num=1, duration=None, method='GET', data=None, ct='text/plain',
auth=None, concurrency=1, headers=None, hook=None):
if headers is None:
headers = {}
if 'content-type' not in headers:
headers['Content-Type'] = ct
if data is not None and data.startswith('py:'):
callable = data[len('py:'):]
data = resolve_name(callable)
