def blockingCallFromThread(reactor, f, *a, **kw):
"""
Run a function in the reactor from a thread, and wait for the result
synchronously, i.e. until the callback chain returned by the function
get a result.
@param reactor: The L{IReactorThreads} provider which will be used to
schedule the function call.
@param f: the callable to run in the reactor thread
@type f: any callable.
@param a: the arguments to pass to C{f}.
@param kw: the keyword arguments to pass to C{f}.
@return: the result of the callback chain.
@raise: any error raised during the callback chain.
"""
queue = Queue.Queue()
def _callFromThread():
result = defer.maybeDeferred(f, *a, **kw)
result.addBoth(queue.put)
reactor.callFromThread(_callFromThread)
result = queue.get()
if isinstance(result, failure.Failure):
result.raiseException()
return result
def _create_derivative_resource_async(self, docid, pdfname):
self._derivs[docid] = PdfDerivatives(
os.path.join(self.dbpath, docid, pdfname),
os.path.join(CACHEROOT, self.dbname, docid))
reactor.callFromThread(
self._with_derivative_resource_sync, docid)
def mineThread(self):
for data in self.qr:
for i in range(data.iterations):
offset = (unpack('I', data.base[i])[0],) if self.GOFFSET else None
self.kernel.search(
self.commandQueue, (data.size, ), (self.WORKSIZE, ),
data.state[0], data.state[1], data.state[2], data.state[3],
data.state[4], data.state[5], data.state[6], data.state[7],
data.state2[1], data.state2[2], data.state2[3],
data.state2[5], data.state2[6], data.state2[7],
data.base[i],
data.f[0], data.f[1], data.f[2], data.f[3],
data.f[4], data.f[5], data.f[6], data.f[7],
self.output_buf, global_offset=offset)
cl.enqueue_read_buffer(self.commandQueue, self.output_buf,
self.output, is_blocking=False)
self.commandQueue.finish()
# The OpenCL code will flag the last item in the output buffer
# when it finds a valid nonce. If that's the case, send it to
# the main thread for postprocessing and clean the buffer
# for the next pass.
if self.output[self.WORKSIZE]:
reactor.callFromThread(self.postprocess,
self.output.copy(), data.nr)
self.output.fill(0)
cl.enqueue_write_buffer(self.commandQueue, self.output_buf,
self.output, is_blocking=False)
def run(self):
while True:
self.userv.wi_available.acquire()
while len(self.userv.wi) == 0:
self.userv.wi_available.wait()
wi = self.userv.wi.pop(0)
if wi == None:
# Shutdown request, relay it further
self.userv.wi.append(None)
self.userv.wi_available.notify()
self.userv.wi_available.release()
if wi == None:
break
command, result_callback, callback_parameters = wi
try:
data, rtpc_delay = self.send_raw(command)
if len(data) == 0:
data, rtpc_delay = None, None
except Exception as e:
print(e)
data, rtpc_delay = None, None
if result_callback != None:
reactor.callFromThread(self.dispatch, result_callback, data, callback_parameters)
if rtpc_delay != None:
reactor.callFromThread(self.userv.register_delay, rtpc_delay)
def onSettingsChange(self, plugin_name, data):
"""
The plugin has been changed by the frontend
:param plugin_name: Name of plugin that changed
:type plugin_name: str
:param data: Complete data (changed and unchanged)
:type data: dict
:rtype: None
"""
logger.info(u"onSettingsChange: {}".format(data))
if not plugin_name:
logger.error("Missing plugin name")
return
if not data:
logger.error("Missing data")
return
logger.info(u"Sending data {}".format(data))
reactor.callFromThread(self._sendJSON, {
'msg': "plugin_data_set",
'plugin_name': plugin_name,
'data': data
})
# trigger db update
reactor.callFromThread(self._sendJSON, {
'msg': "plugin_data_get",
'plugin_name': plugin_name
})
def connect(self, reflect=True, existing_metadata=None):
""" Connect to the database.
When a connection has been established, the
`on_connection_established` is invoked with the metadata.
If connecting fails, the `on_connection_failed` is invoked
with the failure.
:param reflect: If true, reflects the tables and their relations.
:param existing_metadata: If provided, use this existing
metadata instead of creating a metadata instance.
:param reconnect_on_failure: If true, keeps reconnecting if the initial
attempt fails.
"""
if self.is_connected:
return
self._should_reflect = reflect
self._configure_driver()
self._make_metadata(existing_metadata)
try:
self._try_connecting()
except sa.exc.SQLAlchemyError, e:
log.error('Could not connect to database "%s": %s' % (self.database_name, e))
reactor.callFromThread(self.on_connection_failed, failure.Failure())
raise
def _new_fragment(self, hlssink):
self.log("hlsink created a new fragment")
try:
fragment = hlssink.get_property('fragment')
except:
fragment = hlssink.emit('pull-fragment')
reactor.callFromThread(self._process_fragment, fragment)
def blockingCallFromMainThread(f, *a, **kw):
"""
Modified version of twisted.internet.threads.blockingCallFromThread
which waits 30s for results and otherwise assumes the system to be shut down.
This is an ugly workaround for a twisted-internal deadlock.
Please keep the look intact in case someone comes up with a way
to reliably detect from the outside if twisted is currently shutting
down.
"""
queue = Queue.Queue()
def _callFromThread():
result = defer.maybeDeferred(f, *a, **kw)
result.addBoth(queue.put)
reactor.callFromThread(_callFromThread)
result = None
while True:
try:
result = queue.get(True, config.plugins.autotimer.timeout.value*60)
except Queue.Empty as qe:
if True: #not reactor.running: # reactor.running is only False AFTER shutdown, we are during.
doLog("[AutoTimer] Reactor no longer active, aborting.")
else:
break
if isinstance(result, failure.Failure):
result.raiseException()
return result
def disconnect(self):
""" Disconnect from the database.
This closes all active connections in the underlying
connection pool, and calls the `on_connection_lost`. """
self.metadata.bind.dispose()
reactor.callFromThread(self.on_connection_lost)
def check_response(resp):
failed = False
response = resp.content
if resp.status_code == requests.codes.ok:
sha1_regex = re.compile(r'<sha1checksum>([^<]+)<\/sha1checksum>')
sha1_match = sha1_regex.search(response)
if sha1_match is None:
log.err('dshield ERROR: Could not find sha1checksum in response')
failed = True
sha1_local = hashlib.sha1()
sha1_local.update(log_output)
if sha1_match.group(1) != sha1_local.hexdigest():
log.err('dshield ERROR: SHA1 Mismatch {0} {1} .'.format(sha1_match.group(1), sha1_local.hexdigest()))
failed = True
md5_regex = re.compile(r'<md5checksum>([^<]+)<\/md5checksum>')
md5_match = md5_regex.search(response)
if md5_match is None:
log.err('dshield ERROR: Could not find md5checksum in response')
failed = True
md5_local = hashlib.md5()
md5_local.update(log_output)
if md5_match.group(1) != md5_local.hexdigest():
log.err('dshield ERROR: MD5 Mismatch {0} {1} .'.format(md5_match.group(1), md5_local.hexdigest()))
failed = True
log.msg('dshield SUCCESS: Sent {0} bytes worth of data to secure.dshield.org'.format(len(log_output)))
else:
log.err('dshield ERROR: error {0}.'.format(resp.status_code))
log.err('Response was {0}'.format(response))
failed = True
if failed:
# Something went wrong, we need to add them to batch.
reactor.callFromThread(self.transmission_error, batch)
def test(self):
crawler = mock.MagicMock()
crawler.settings = CrawlerSettings()
crawler.settings.overrides['USER_AGENT'] = 'CustomAgent'
self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler)
crawler.settings.overrides['ROBOTSTXT_OBEY'] = True
crawler.engine.download = mock.MagicMock()
ROBOTS = re.sub(r'^\s+(?m)', '', '''
User-Agent: *
Disallow: /admin/
Disallow: /static/
''')
response = Response('http://site.local/robots.txt', body=ROBOTS)
def return_response(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.callback, response)
return deferred
crawler.engine.download.side_effect = return_response
middleware = RobotsTxtMiddleware(crawler)
spider = None # not actually used
# There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously,
# and it is actually fetched only *after* first process_request completes.
# So, first process_request will always succeed.
# We defer test() because otherwise robots.txt download mock will be called after assertRaises failure.
self.assertIsNone(middleware.process_request(Request('http://site.local'), spider)) # not affected by robots.txt
def test(r):
self.assertIsNone(middleware.process_request(Request('http://site.local/allowed'), spider))
self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/admin/main'), spider)
self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/static/'), spider)
deferred = Deferred()
deferred.addCallback(test)
reactor.callFromThread(deferred.callback, None)
return deferred
def do_list(self, line):
"""List data about a room.
"list members <roomid> [query]" - List all the members in this room.
"list messages <roomid> [query]" - List all the messages in this room.
Where [query] will be directly applied as query parameters, allowing
you to use the pagination API. E.g. the last 3 messages in this room:
"list messages <roomid> from=END&to=START&limit=3"
"""
args = self._parse(line, ["type", "roomid", "qp"])
if not "type" in args or not "roomid" in args:
print "Must specify type and room ID."
return
if args["type"] not in ["members", "messages"]:
print "Unrecognised type: %s" % args["type"]
return
room_id = args["roomid"]
path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
qp = {"access_token": self._tok()}
if "qp" in args:
for key_value_str in args["qp"].split("&"):
try:
key_value = key_value_str.split("=")
qp[key_value[0]] = key_value[1]
except:
print "Bad query param: %s" % key_value
return
reactor.callFromThread(self._run_and_pprint, "GET", path,
query_params=qp)
def _send_receipt(self, event, feedback_type):
path = ("/rooms/%s/messages/%s/%s/feedback/%s/%s" %
(urllib.quote(event["room_id"]), event["user_id"], event["msg_id"],
self._usr(), feedback_type))
data = {}
reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data,
alt_text="Sent receipt for %s" % event["msg_id"])
def do_joinalias(self, line):
try:
args = self._parse(line, ["roomname"], force_keys=True)
path = "/join/%s" % urllib.quote(args["roomname"])
reactor.callFromThread(self._run_and_pprint, "POST", path, {})
except Exception as e:
print e
def do_topic(self, line):
""""topic [set|get] <roomid> [<newtopic>]"
Set the topic for a room: topic set <roomid> <newtopic>
Get the topic for a room: topic get <roomid>
"""
try:
args = self._parse(line, ["action", "roomid", "topic"])
if "action" not in args or "roomid" not in args:
print "Must specify set|get and a room ID."
return
if args["action"].lower() not in ["set", "get"]:
print "Must specify set|get, not %s" % args["action"]
return
path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
if args["action"].lower() == "set":
if "topic" not in args:
print "Must specify a new topic."
return
body = {
"topic": args["topic"]
}
reactor.callFromThread(self._run_and_pprint, "PUT", path, body)
elif args["action"].lower() == "get":
reactor.callFromThread(self._run_and_pprint, "GET", path)
except Exception as e:
print e
def negotiateSession(self, username, tryNext=False):
"""
Negotiate through the instant messaging layer a direct peer-to-peer connection
with the user that has the given username. Note the username in question must
represent another SubliminalCollaborator Negotiator instance used by another
user.
If the user identified by the given username is not in the C{Array} returned
by listUsers(), the expectation is that successful execution of this function
will result in the given username being added to the list of known users.
"""
if (not username in self.peerUsers) and (not username in self.unverifiedUsers):
self.addUserToLists(username)
if not tryNext:
self.hostAddressToTryQueue = socket.gethostbyname_ex(socket.gethostname())[2]
if len(self.hostAddressToTryQueue) == 0:
status_bar.status_message('failed to share with %s' % username)
logger.warn('Unable to connect to peer %s, all host addresses tried and failed!' % username)
# TODO error reporting in UI
self.msg(username, 'NO-GOOD-HOST-IP')
return
ipaddress = self.hostAddressToTryQueue.pop()
session = base.BasePeer(username)
port = session.hostConnect()
status_bar.status_message('trying to share with %s@%s' % (username, ipaddress))
logger.debug('Negotiating collab session with %s with ip address %s on port %d' % (username, ipaddress, port))
reactor.callFromThread(self.ctcpMakeQuery, username, [('DCC CHAT', 'collaborate %s %d' % (ipaddress, port))])
self.negotiateCallback(session)
def do_register(self, line):
"""Registers for a new account: "register <userid> <noupdate>"
<userid> : The desired user ID
<noupdate> : Do not automatically clobber config values.
"""
args = self._parse(line, ["userid", "noupdate"])
password = None
pwd = None
pwd2 = "_"
while pwd != pwd2:
pwd = getpass.getpass("Type a password for this user: "******"Retype the password: "******"Password mismatch."
pwd = None
else:
password = pwd
body = {
"type": "m.login.password"
}
if "userid" in args:
body["user"] = args["userid"]
if password:
body["password"] = password
reactor.callFromThread(self._do_register, body,
"noupdate" not in args)
def run(self):
while self.port_agent.keep_going:
if not self.port_agent._pause:
reactor.callFromThread(self.port_agent.router.got_data, get_one(self.orb))
time.sleep(.0001)
else:
time.sleep(.001)
def blockingCallFromThread(reactor, f, *a, **kw):
"""
Run a function in the reactor from a thread, and wait for the result
synchronously. If the function returns a L{Deferred}, wait for its
result and return that.
@param reactor: The L{IReactorThreads} provider which will be used to
schedule the function call.
@param f: the callable to run in the reactor thread
@type f: any callable.
@param a: the arguments to pass to C{f}.
@param kw: the keyword arguments to pass to C{f}.
@return: the result of the L{Deferred} returned by C{f}, or the result
of C{f} if it returns anything other than a L{Deferred}.
@raise: If C{f} raises a synchronous exception,
C{blockingCallFromThread} will raise that exception. If C{f}
returns a L{Deferred} which fires with a L{Failure},
C{blockingCallFromThread} will raise that failure's exception (see
L{Failure.raiseException}).
"""
queue = Queue.Queue()
def _callFromThread():
result = defer.maybeDeferred(f, *a, **kw)
result.addBoth(queue.put)
reactor.callFromThread(_callFromThread)
result = queue.get()
if isinstance(result, failure.Failure):
result.raiseException()
return result
def _signal_kill(self, signum, _):
signame = signal_names[signum]
log.msg('Received %s twice, forcing unclean shutdown' % signame, \
level=log.INFO)
log.log_level = log.SILENT # disable logging of confusing tracebacks
reactor.callFromThread(self.engine.kill)
install_shutdown_handlers(signal.SIG_IGN)
def processDone(self, view, protocol, client,
account, request):
if __debug__:
trace("processDone")
if self.shuttingDown:
return None
# request
# 0: cmd
# 1: client
# 2: accountUUID
dt = self.getDownloadTracker(account)
if dt.totalDownloaded > 0:
# This is a PyICU.ChoiceFormat class
txt = constants.DOWNLOAD_CHANDLER_MESSAGES.format(dt.totalDownloaded)
setStatusMessage(txt % \
{'accountName': account.displayName,
'numberTotal': dt.totalDownloaded,
'numberNew': dt.totalNewDownloaded,
'numberUpdates': dt.totalUpdateDownloaded,
'numberDuplicates': dt.totalIgnoreDownloaded,
'numberErrors': dt.totalErrorDownloaded})
else:
setStatusMessage(constants.DOWNLOAD_NO_MESSAGES % \
{'accountName': account.displayName})
self._resetWorker(account)
# Post a notification to the Mail Protocol Client
# That the requested actions are finished.
reactor.callFromThread(client.requestsComplete)
def intermediateWrite(self, timers, conflicting, similar, skipped):
returnlist = []
extend = returnlist.extend
for (name, begin, end, serviceref, autotimername, message) in timers:
ref = ServiceReference(str(serviceref))
extend(
(
"<e2simulatedtimer>\n" " <e2servicereference>",
stringToXML(serviceref),
"</e2servicereference>\n",
" <e2servicename>",
stringToXML(ref.getServiceName().replace("\xc2\x86", "").replace("\xc2\x87", "")),
"</e2servicename>\n",
" <e2name>",
stringToXML(name),
"</e2name>\n",
" <e2timebegin>",
str(begin),
"</e2timebegin>\n",
" <e2timeend>",
str(end),
"</e2timeend>\n",
" <e2autotimername>",
stringToXML(autotimername),
"</e2autotimername>\n" "</e2simulatedtimer>\n",
)
)
if self._stillAlive:
reactor.callFromThread(lambda: self._req.write("".join(returnlist)))
def wrapper(*args, **kargs):
q = Queue()
def callback(value):
q.put(None)
def errback(failure):
# Retrieve and save full exception info
try:
failure.raiseException()
except:
q.put(sys.exc_info())
def g():
try:
d = func(*args, **kargs)
try:
d.addCallbacks(callback, errback)
# Check for a common mistake and display a nice error
# message
except AttributeError:
raise TypeError("you must return a twisted Deferred "
"from your test case!")
# Catch exceptions raised in the test body (from the
# Twisted thread)
except:
q.put(sys.exc_info())
reactor.callFromThread(g)
try:
error = q.get(timeout=timeout)
except Empty:
raise TimeExpired("timeout expired before end of test (%f s.)"
% timeout)
# Re-raise all exceptions
if error is not None:
exc_type, exc_value, tb = error
raise exc_type(exc_value).with_traceback(tb)
def run(self):
maxemptydata = 100
while True:
try:
data, address = self.userv.skt.recvfrom(8192)
if not data:
# Ugly hack to detect socket being closed under us on Linux.
# The problem is that even call on non-closed socket can
# sometimes return empty data buffer, making AsyncReceiver
# to exit prematurely.
maxemptydata -= 1
if maxemptydata == 0:
break
continue
else:
maxemptydata = 100
except Exception, why:
if isinstance(why, socket.error) and why[0] in (ECONNRESET, ENOTCONN, ESHUTDOWN):
break
if isinstance(why, socket.error) and why[0] in (EINTR,):
continue
else:
print datetime.now(), 'Udp_server: unhandled exception when receiving incoming data'
print '-' * 70
traceback.print_exc(file = sys.stdout)
print '-' * 70
sys.stdout.flush()
sleep(1)
continue
rtime = MonoTime()
if self.userv.uopts.family == socket.AF_INET6:
address = ('[%s]' % address[0], address[1])
reactor.callFromThread(self.userv.handle_read, data, address, rtime)
def runTestFunc():
no_errors = False
try:
func(*clients)
no_errors = True
finally:
reactor.callFromThread(superviseFunc, connections, no_errors)
def _signal_kill(self, signum, _):
install_shutdown_handlers(signal.SIG_IGN)
signame = signal_names[signum]
log.msg(format='Received %(signame)s twice, forcing unclean shutdown',
level=log.INFO, signame=signame)
self._stop_logging()
reactor.callFromThread(self._stop_reactor)
def _client_removed_cb(self, element, arg0, client_status):
# treat as error if we were removed because of GST_CLIENT_STATUS_ERROR
# FIXME: can we use the symbol instead of a numeric constant ?
if client_status == 4:
# since we get called from the streaming thread, hand off handling
# to the reactor's thread
reactor.callFromThread(self._client_error_cb)
def writeResponse(message):
global processing, queue
self.transport.write(message + "\r\n")
if queue.empty() == False:
reactor.callFromThread(self.lineReceived, queue.get())
print "Processing message in Queue"
processing = False
def next(self):
"""Since QueueReader is iterable, this is the function that runs the
for-loop and dispatches work to the thread.
This should be the only thread that executes outside of the Twisted
main thread.
"""
# If we just completed a range, we should tell the main thread.
now = time()
if self.currentData:
dt = now - self.startedAt
# self.currentData[1] is the un-preprocessed NonceRange.
reactor.callFromThread(self._ranExecution, dt, self.currentData[1])
self.startedAt = now
# Block for more data from the main thread. In 99% of cases, though,
# there should already be something here.
# Note that this comes back with either a tuple, or a StopIteration()
self.currentData = self.dataQueue.get(True)
# Does the main thread want us to shut down, or pass some more data?
if isinstance(self.currentData, StopIteration):
raise self.currentData
# We just took the only item in the queue. It needs to be restocked.
reactor.callFromThread(self._requestMore)
# currentData is actually a tuple, with item 0 intended for the kernel.
return self.currentData[0]
def run(self):
req = self._req
if self._stillAlive:
req.setResponseCode(http.OK)
req.setHeader("Content-type", "application/xhtml+xml")
req.setHeader("charset", "UTF-8")
reactor.callFromThread(
lambda: req.write(
'<?xml version="1.0" encoding="UTF-8" ?>\n<e2autotimersimulate api_version="'
+ str(API_VERSION)
+ '">\n'
)
)
def finishRequest():
req.write("</e2autotimersimulate>")
req.finish()
id = req.args.get("id")
if id:
self.id = int(id[0])
else:
self.id = None
try:
autotimer.parseEPG(simulateOnly=True, uniqueId=self.id, callback=self.intermediateWrite)
except Exception as e:
def finishRequest():
req.write("<exception>" + str(e) + "</exception><|PURPOSEFULLYBROKENXML<")
req.finish()
if self._stillAlive:
reactor.callFromThread(finishRequest)
def main(args, net, datadir_path, merged_urls, worker_endpoint):
try:
print 'p2pool (version %s)' % (p2pool.__version__, )
print
@defer.inlineCallbacks
def connect_p2p():
# connect to bitcoind over bitcoin-p2p
print '''Testing bitcoind P2P connection to '%s:%s'...''' % (
args.bitcoind_address, args.bitcoind_p2p_port)
factory = bitcoin_p2p.ClientFactory(net.PARENT)
reactor.connectTCP(args.bitcoind_address, args.bitcoind_p2p_port,
factory)
yield factory.getProtocol() # waits until handshake is successful
print ' ...success!'
print
defer.returnValue(factory)
if args.testnet: # establish p2p connection first if testnet so bitcoind can work without connections
factory = yield connect_p2p()
# connect to bitcoind over JSON-RPC and do initial getmemorypool
url = '%s://%s:%i/' % ('https' if args.bitcoind_rpc_ssl else 'http',
args.bitcoind_address, args.bitcoind_rpc_port)
print '''Testing bitcoind RPC connection to '%s' with username '%s'...''' % (
url, args.bitcoind_rpc_username)
bitcoind = jsonrpc.Proxy(
url,
dict(Authorization='Basic ' +
base64.b64encode(args.bitcoind_rpc_username + ':' +
args.bitcoind_rpc_password)),
timeout=30)
yield helper.check(bitcoind, net)
temp_work = yield helper.getwork(bitcoind)
bitcoind_warning_var = variable.Variable(None)
@defer.inlineCallbacks
def poll_warnings():
errors = (yield
deferral.retry('Error while calling getmininginfo:')(
bitcoind.rpc_getmininginfo)())['errors']
bitcoind_warning_var.set(errors if errors != '' else None)
yield poll_warnings()
task.LoopingCall(poll_warnings).start(20 * 60)
print ' ...success!'
print ' Current block hash: %x' % (temp_work['previous_block'], )
print ' Current block height: %i' % (temp_work['height'] - 1, )
print
if not args.testnet:
factory = yield connect_p2p()
print 'Determining payout address...'
if args.pubkey_hash is None:
address_path = os.path.join(datadir_path, 'cached_payout_address')
if os.path.exists(address_path):
with open(address_path, 'rb') as f:
address = f.read().strip('\r\n')
print ' Loaded cached address: %s...' % (address, )
else:
address = None
if address is not None:
res = yield deferral.retry(
'Error validating cached address:',
5)(lambda: bitcoind.rpc_validateaddress(address))()
if not res['isvalid'] or not res['ismine']:
print ' Cached address is either invalid or not controlled by local bitcoind!'
address = None
if address is None:
print ' Getting payout address from bitcoind...'
address = yield deferral.retry(
'Error getting payout address from bitcoind:',
5)(lambda: bitcoind.rpc_getaccountaddress('p2pool'))()
with open(address_path, 'wb') as f:
f.write(address)
my_pubkey_hash = bitcoin_data.address_to_pubkey_hash(
address, net.PARENT)
else:
my_pubkey_hash = args.pubkey_hash
print ' ...success! Payout address:', bitcoin_data.pubkey_hash_to_address(
my_pubkey_hash, net.PARENT)
print
ss = p2pool_data.ShareStore(os.path.join(datadir_path, 'shares.'), net)
shares = {}
known_verified = set()
print "Loading shares..."
for i, (mode, contents) in enumerate(ss.get_shares()):
if mode == 'share':
contents.time_seen = 0
shares[contents.hash] = contents
if len(shares) % 1000 == 0 and shares:
print " %i" % (len(shares), )
elif mode == 'verified_hash':
known_verified.add(contents)
else:
raise AssertionError()
print " ...done loading %i shares (%i verified)!" % (
len(shares), len(known_verified))
print
print 'Initializing work...'
node = p2pool_node.Node(factory, bitcoind, shares.values(),
known_verified, net)
yield node.start()
for share_hash in shares:
if share_hash not in node.tracker.items:
ss.forget_share(share_hash)
for share_hash in known_verified:
if share_hash not in node.tracker.verified.items:
ss.forget_verified_share(share_hash)
del shares, known_verified
node.tracker.removed.watch(lambda share: ss.forget_share(share.hash))
node.tracker.verified.removed.watch(
lambda share: ss.forget_verified_share(share.hash))
def save_shares():
for share in node.tracker.get_chain(
node.best_share_var.value,
min(node.tracker.get_height(node.best_share_var.value),
2 * net.CHAIN_LENGTH)):
ss.add_share(share)
if share.hash in node.tracker.verified.items:
ss.add_verified_hash(share.hash)
task.LoopingCall(save_shares).start(60)
print ' ...success!'
print
print 'Joining p2pool network using port %i...' % (args.p2pool_port, )
@defer.inlineCallbacks
def parse(x):
if ':' in x:
ip, port = x.split(':')
defer.returnValue(((yield reactor.resolve(ip)), int(port)))
else:
defer.returnValue(((yield reactor.resolve(x)), net.P2P_PORT))
addrs = {}
if os.path.exists(os.path.join(datadir_path, 'addrs')):
try:
with open(os.path.join(datadir_path, 'addrs'), 'rb') as f:
addrs.update(
dict((tuple(k), v) for k, v in json.loads(f.read())))
except:
print >> sys.stderr, 'error parsing addrs'
for addr_df in map(parse, net.BOOTSTRAP_ADDRS):
try:
addr = yield addr_df
if addr not in addrs:
addrs[addr] = (0, time.time(), time.time())
except:
log.err()
connect_addrs = set()
for addr_df in map(parse, args.p2pool_nodes):
try:
connect_addrs.add((yield addr_df))
except:
log.err()
node.p2p_node = p2pool_node.P2PNode(
node,
port=args.p2pool_port,
max_incoming_conns=args.p2pool_conns,
addr_store=addrs,
connect_addrs=connect_addrs,
desired_outgoing_conns=args.p2pool_outgoing_conns,
)
node.p2p_node.start()
def save_addrs():
with open(os.path.join(datadir_path, 'addrs'), 'wb') as f:
f.write(json.dumps(node.p2p_node.addr_store.items()))
task.LoopingCall(save_addrs).start(60)
print ' ...success!'
print
if args.upnp:
@defer.inlineCallbacks
def upnp_thread():
while True:
try:
is_lan, lan_ip = yield ipdiscover.get_local_ip()
if is_lan:
pm = yield portmapper.get_port_mapper()
yield pm._upnp.add_port_mapping(
lan_ip, args.p2pool_port, args.p2pool_port,
'p2pool', 'TCP')
except defer.TimeoutError:
pass
except:
if p2pool.DEBUG:
log.err(None, 'UPnP error:')
yield deferral.sleep(random.expovariate(1 / 120))
upnp_thread()
# start listening for workers with a JSON-RPC server
print 'Listening for workers on %r port %i...' % (worker_endpoint[0],
worker_endpoint[1])
wb = work.WorkerBridge(node, my_pubkey_hash, args.donation_percentage,
merged_urls, args.worker_fee)
web_root = web.get_web_root(wb, datadir_path, bitcoind_warning_var)
worker_interface.WorkerInterface(wb).attach_to(
web_root, get_handler=lambda request: request.redirect('/static/'))
deferral.retry('Error binding to worker port:', traceback=False)(
reactor.listenTCP)(worker_endpoint[1],
server.Site(web_root),
interface=worker_endpoint[0])
with open(os.path.join(os.path.join(datadir_path, 'ready_flag')),
'wb') as f:
pass
print ' ...success!'
print
# done!
print 'Started successfully!'
print 'Go to http://127.0.0.1:%i/ to view graphs and statistics!' % (
worker_endpoint[1], )
if args.donation_percentage > 0.51:
print '''Donating %.1f%% of work towards P2Pool's development. Thanks for the tip!''' % (
args.donation_percentage, )
elif args.donation_percentage < 0.49:
print '''Donating %.1f%% of work towards P2Pool's development. Please donate to encourage further development of P2Pool!''' % (
args.donation_percentage, )
else:
print '''Donating %.1f%% of work towards P2Pool's development. Thank you!''' % (
args.donation_percentage, )
print 'You can increase this amount with --give-author argument! (or decrease it, if you must)'
print
if hasattr(signal, 'SIGALRM'):
signal.signal(
signal.SIGALRM, lambda signum, frame: reactor.callFromThread(
sys.stderr.write, 'Watchdog timer went off at:\n' + ''.
join(traceback.format_stack())))
signal.siginterrupt(signal.SIGALRM, False)
task.LoopingCall(signal.alarm, 30).start(1)
if args.irc_announce:
from twisted.words.protocols import irc
class IRCClient(irc.IRCClient):
nickname = 'p2pool%02i' % (random.randrange(100), )
channel = net.ANNOUNCE_CHANNEL
def lineReceived(self, line):
if p2pool.DEBUG:
print repr(line)
irc.IRCClient.lineReceived(self, line)
def signedOn(self):
self.in_channel = False
irc.IRCClient.signedOn(self)
self.factory.resetDelay()
self.join(self.channel)
@defer.inlineCallbacks
def new_share(share):
if not self.in_channel:
return
if share.pow_hash <= share.header[
'bits'].target and abs(share.timestamp -
time.time()) < 10 * 60:
yield deferral.sleep(random.expovariate(1 / 60))
message = '\x02%s BLOCK FOUND by %s! %s%064x' % (
net.NAME.upper(),
bitcoin_data.script2_to_address(
share.new_script, net.PARENT),
net.PARENT.BLOCK_EXPLORER_URL_PREFIX,
share.header_hash)
if all('%x' %
(share.header_hash, ) not in old_message
for old_message in self.recent_messages):
self.say(self.channel, message)
self._remember_message(message)
self.watch_id = node.tracker.verified.added.watch(
new_share)
self.recent_messages = []
def joined(self, channel):
self.in_channel = True
def left(self, channel):
self.in_channel = False
def _remember_message(self, message):
self.recent_messages.append(message)
while len(self.recent_messages) > 100:
self.recent_messages.pop(0)
def privmsg(self, user, channel, message):
if channel == self.channel:
self._remember_message(message)
def connectionLost(self, reason):
node.tracker.verified.added.unwatch(self.watch_id)
print 'IRC connection lost:', reason.getErrorMessage()
class IRCClientFactory(protocol.ReconnectingClientFactory):
protocol = IRCClient
reactor.connectTCP("irc.freenode.net", 6667, IRCClientFactory())
@defer.inlineCallbacks
def status_thread():
last_str = None
last_time = 0
while True:
yield deferral.sleep(3)
try:
height = node.tracker.get_height(node.best_share_var.value)
this_str = 'P2Pool: %i shares in chain (%i verified/%i total) Peers: %i (%i incoming)' % (
height,
len(node.tracker.verified.items),
len(node.tracker.items),
len(node.p2p_node.peers),
sum(1 for peer in node.p2p_node.peers.itervalues()
if peer.incoming),
) + (' FDs: %i R/%i W' %
(len(reactor.getReaders()), len(reactor.getWriters()))
if p2pool.DEBUG else '')
datums, dt = wb.local_rate_monitor.get_datums_in_last()
my_att_s = sum(datum['work'] / dt for datum in datums)
this_str += '\n Local: %sH/s in last %s Local dead on arrival: %s Expected time to share: %s' % (
math.format(int(my_att_s)),
math.format_dt(dt),
math.format_binomial_conf(
sum(1 for datum in datums if datum['dead']),
len(datums), 0.95),
math.format_dt(2**256 / node.tracker.items[
node.best_share_var.value].max_target / my_att_s)
if my_att_s and node.best_share_var.value else '???',
)
if height > 2:
(stale_orphan_shares,
stale_doa_shares), shares, _ = wb.get_stale_counts()
stale_prop = p2pool_data.get_average_stale_prop(
node.tracker, node.best_share_var.value,
min(60 * 60 // net.SHARE_PERIOD, height))
real_att_s = p2pool_data.get_pool_attempts_per_second(
node.tracker, node.best_share_var.value,
min(height - 1, 60 * 60 //
net.SHARE_PERIOD)) / (1 - stale_prop)
this_str += '\n Shares: %i (%i orphan, %i dead) Stale rate: %s Efficiency: %s Current payout: %.4f %s' % (
shares,
stale_orphan_shares,
stale_doa_shares,
math.format_binomial_conf(
stale_orphan_shares + stale_doa_shares, shares,
0.95),
math.format_binomial_conf(
stale_orphan_shares + stale_doa_shares, shares,
0.95, lambda x: (1 - x) / (1 - stale_prop)),
node.get_current_txouts().get(
bitcoin_data.pubkey_hash_to_script2(
my_pubkey_hash), 0) * 1e-8,
net.PARENT.SYMBOL,
)
this_str += '\n Pool: %sH/s Stale rate: %.1f%% Expected time to block: %s' % (
math.format(int(real_att_s)),
100 * stale_prop,
math.format_dt(
2**256 /
node.bitcoind_work.value['bits'].target /
real_att_s),
)
for warning in p2pool_data.get_warnings(
node.tracker, node.best_share_var.value, net,
bitcoind_warning_var.value,
node.bitcoind_work.value):
print >> sys.stderr, '#' * 40
print >> sys.stderr, '>>> Warning: ' + warning
print >> sys.stderr, '#' * 40
if this_str != last_str or time.time() > last_time + 15:
print this_str
last_str = this_str
last_time = time.time()
except:
log.err()
status_thread()
except:
reactor.stop()
log.err(None, 'Fatal error:')
def shutdown_twisted():
logger.info("Stopping Twisted")
yield reactor.callFromThread(reactor.stop)
def return_response(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.callback, response)
return deferred
def remote_control(self, port):
s = socket.socket() # Create a socket object
s.bind(('', port)) # Bind to the port
s.listen(5) # Now wait for client connection.
logger.info('Remote control is listening on {}:{}'.format(
socket.getfqdn(), port))
while True:
c, addr = s.accept() # Establish connection with client.
logger.info('Got connection from {}'.format(addr))
self._dmrgui = addr[0]
_tmp = c.recv(1024)
_tmp = _tmp.split(None)[0] #first get rid of whitespace
_cmd = _tmp.split('=')[0]
logger.info('Command:"{}"'.format(_cmd))
if _cmd:
if _cmd == 'reread_subscribers':
reread_subscribers()
elif _cmd == 'reread_config':
self.readConfigFile(self._configFile, None,
self._currentNetwork)
elif _cmd == 'txTg':
self._tx_tg = hex_str_3(int(_tmp.split('=')[1]))
print('New txTg = ' + str(int_id(self._tx_tg)))
elif _cmd == 'txTs':
self._tx_ts = int(_tmp.split('=')[1])
print('New txTs = ' + str(self._tx_ts))
elif _cmd == 'section':
self.readConfigFile(self._configFile, _tmp.split('=')[1])
elif _cmd == 'gateway_dmr_id':
self._gateway_dmr_id = int(_tmp.split('=')[1])
print('New gateway_dmr_id = ' + str(self._gateway_dmr_id))
elif _cmd == 'gateway_peer_id':
peerID = int(_tmp.split('=')[1])
self._config['LOCAL']['RADIO_ID'] = hex_str_3(peerID)
print('New peer_id = ' + str(peerID))
elif _cmd == 'restart':
reactor.callFromThread(reactor.stop)
elif _cmd == 'playbackFromFile':
self.playbackFromFile('ambe.bin')
elif _cmd == 'tgs':
_args = _tmp.split('=')[1]
self._tg_filter = map(int, _args.split(','))
logger.info('New TGs={}'.format(self._tg_filter))
elif _cmd == 'dump_template':
self.dumpTemplate('PrivateVoice.bin')
elif _cmd == 'get_alias':
self._sock.sendto(
'reply dmr_info {} {} {} {}'.format(
self._currentNetwork,
int_id(self._CONFIG[self._currentNetwork]['LOCAL']
['RADIO_ID']), self._gateway_dmr_id,
get_subscriber_info(hex_str_3(
self._gateway_dmr_id))), (self._dmrgui, 34003))
elif _cmd == 'eval':
_sz = len(_tmp) - 5
_evalExpression = _tmp[-_sz:]
_evalResult = eval(_evalExpression)
print("eval of {} is {}".format(_evalExpression,
_evalResult))
self._sock.sendto('reply eval {}'.format(_evalResult),
(self._dmrgui, 34003))
elif _cmd == 'exec':
_sz = len(_tmp) - 5
_evalExpression = _tmp[-_sz:]
exec(_evalExpression)
print("exec of {}".format(_evalExpression))
else:
logger.error('Unknown command')
c.close() # Close the connection
def stop(self):
reactor.callFromThread(reactor.stop)
def rotateLog(signal, frame):
from twisted.internet import reactor
reactor.callFromThread(logFile.rotate)
def add_timer(self, timer):
self._timers.add_timer(timer)
# callFromThread to schedule from the loop thread, where
# the timeout task can safely be modified
reactor.callFromThread(self._schedule_timeout, timer.end)
def main():
global config
parser = argparse.ArgumentParser(
description=
'mitm6 - pwning IPv4 via IPv6\nFor help or reporting issues, visit https://github.com/fox-it/mitm6',
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("-i",
"--interface",
type=str,
metavar='INTERFACE',
help="Interface to use (default: autodetect)")
parser.add_argument(
"-l",
"--localdomain",
type=str,
metavar='LOCALDOMAIN',
help=
"Domain name to use as DNS search domain (default: use first DNS domain)"
)
parser.add_argument(
"-4",
"--ipv4",
type=str,
metavar='ADDRESS',
help="IPv4 address to send packets from (default: autodetect)")
parser.add_argument(
"-6",
"--ipv6",
type=str,
metavar='ADDRESS',
help=
"IPv6 link-local address to send packets from (default: autodetect)")
parser.add_argument(
"-m",
"--mac",
type=str,
metavar='ADDRESS',
help=
"Custom mac address - probably breaks stuff (default: mac of selected interface)"
)
parser.add_argument(
"-a",
"--no-ra",
action='store_true',
help=
"Do not advertise ourselves (useful for networks which detect rogue Router Advertisements)"
)
parser.add_argument("-v",
"--verbose",
action='store_true',
help="Show verbose information")
parser.add_argument("--debug",
action='store_true',
help="Show debug information")
filtergroup = parser.add_argument_group("Filtering options")
filtergroup.add_argument(
"-d",
"--domain",
action='append',
default=[],
metavar='DOMAIN',
help=
"Domain name to filter DNS queries on (Whitelist principle, multiple can be specified.)"
)
filtergroup.add_argument(
"-b",
"--blacklist",
action='append',
default=[],
metavar='DOMAIN',
help=
"Domain name to filter DNS queries on (Blacklist principle, multiple can be specified.)"
)
filtergroup.add_argument(
"-hw",
"--host-whitelist",
action='append',
default=[],
metavar='DOMAIN',
help=
"Hostname (FQDN) to filter DHCPv6 queries on (Whitelist principle, multiple can be specified.)"
)
filtergroup.add_argument(
"-hb",
"--host-blacklist",
action='append',
default=[],
metavar='DOMAIN',
help=
"Hostname (FQDN) to filter DHCPv6 queries on (Blacklist principle, multiple can be specified.)"
)
filtergroup.add_argument(
"--ignore-nofqdn",
action='store_true',
help=
"Ignore DHCPv6 queries that do not contain the Fully Qualified Domain Name (FQDN) option."
)
args = parser.parse_args()
config = Config(args)
print('Starting mitm6 using the following configuration:')
print('Primary adapter: %s [%s]' % (config.default_if, config.selfmac))
print('IPv4 address: %s' % config.selfipv4)
print('IPv6 address: %s' % config.selfaddr)
if config.localdomain is not None:
print('DNS local search domain: %s' % config.localdomain)
if not config.dns_whitelist and not config.dns_blacklist:
print(
'Warning: Not filtering on any domain, mitm6 will reply to all DNS queries.\nUnless this is what you want, specify at least one domain with -d'
)
else:
if not config.dns_whitelist:
print('DNS whitelist: *')
else:
print('DNS whitelist: %s' % ', '.join(config.dns_whitelist))
if config.dns_blacklist:
print('DNS blacklist: %s' % ', '.join(config.dns_blacklist))
if config.host_whitelist:
print('Hostname whitelist: %s' % ', '.join(config.host_whitelist))
if config.host_blacklist:
print('Hostname blacklist: %s' % ', '.join(config.host_blacklist))
#Main packet capture thread
d = threads.deferToThread(
sniff,
filter="ip6 proto \\udp or arp or udp port 53",
prn=lambda x: reactor.callFromThread(parsepacket, x),
stop_filter=should_stop)
d.addErrback(print_err)
#RA loop
if not args.no_ra:
loop = task.LoopingCall(send_ra)
d = loop.start(30.0)
d.addErrback(print_err)
# Set up DNS
dnssock = setupFakeDns()
reactor.adoptDatagramPort(dnssock.fileno(), socket.AF_INET6,
DatagramProtocol())
reactor.addSystemEventTrigger('before', 'shutdown', shutdownnotice)
reactor.run()
def onResult(success, counter):
reactor.callFromThread(submit, counter)
def _signal_kill(self, signum, _):
install_shutdown_handlers(signal.SIG_IGN)
signame = signal_names[signum]
logger.info('Received %(signame)s twice, forcing unclean shutdown',
{'signame': signame})
reactor.callFromThread(self._stop_reactor)
def mine_block(self):
while True:
if self.block_chain_uptodate or self.genesis:
# mine block
if self.block_chain is not None:
txs = self.tx_handler.org_txs_for_block(
self.transaction_pool, self.tx_fee_pool)
block = self.block_chain.mine_block(txs,
self.node_name +
"'s message",
coin_base=100,
send_addr=None)
# if new block received before mining this then. start over
if self.new_block_received:
self.new_block_received = False
continue
if self.block_chain.add_block_to_chain(block):
with self.lock:
tx_coinbase = txs[-1]
self.utxopool.add_to_pool(UTXO(
tx_coinbase.txid, 0),
tx_coinbase.txOuts[0],
txtype='coinbase')
#chk if there is any fee tx
for tx in txs:
if tx.txtype == "FEE":
self.utxopool.add_to_pool(UTXO(tx.txid, 0),
tx.txOuts[0],
txtype='fee')
print
self.add_to_conf_utxopool(block)
self.wallet_manager.utxopool = self.utxopool
self.wallet_manager.conf_utxopool = self.confirmed_utxopool
self.wallet_manager.update_utxopool_on_wallet()
self.wallet_manager.update_all_wallets_account_balance(
)
for t in txs[:-1]:
try:
self.transaction_pool.remove(t)
except:
continue
print("Block Mined...")
self.Portal.update_debugger("Block Mined...")
reactor.callFromThread(self.send_block_via_clients,
block)
with open("{}_blockchain.bc".format(self.node_name),
"wb") as handle:
pickle.dump(self.block_chain, handle)
chain_len = len(self.block_chain.chain)
size = os.path.getsize("{}_blockchain.bc".format(
self.node_name))
num_utxos = len(self.utxopool.utxos)
self.num_blk_mined += 1
self.num_txs += len(txs)
self.Portal.update_blockchain_info(
chain_len, int(size), self.num_txs, num_utxos,
self.num_blk_mined)
self.Portal.update_blockchain_gui(
block.index, block.hash)
time.sleep(10)
def _probe_cb(self, pad, event):
if event.type == gst.EVENT_EOS:
self.debug('eos received, stopping')
reactor.callFromThread(self.callback, "eos")
def _handle_SIGHUP(self, *args):
reactor.callFromThread(self.do_rotate)
def waiter(th):
th.join()
Shipper.shutdown()
reactor.callFromThread(reactor.stop)
def _callFromThreadCallback(self, d):
reactor.callFromThread(self._callFromThreadCallback2, d)
reactor.callLater(0, self._stopCallFromThreadCallback)
def add_block_to_chain(self, block, owner):
block = pickle.loads(block)
if owner != self.node.node_name:
print("New block Received...")
self.node.Portal.update_debugger("New block Received...")
#validate the txs in block
valid_txs, fees = self.node.tx_handler.handle_txs(
block.txs,
self.node.transaction_pool,
tx_fee_pool=self.node.tx_fee_pool)
if len(valid_txs) != len(block.txs):
print("Block Contains Invalid Transactions. Block rejected!!")
return
if self.node.block_chain.add_block_to_chain(block):
with open("{}_blockchain.bc".format(self.node.node_name),
"wb") as handle:
pickle.dump(self.node.block_chain, handle)
print("Block Added To Blockchain...")
print("Forwarding Block To Peers...")
self.node.Portal.update_debugger(
"Block Added To Blockchain...")
self.node.Portal.update_debugger(
"Forwarding Block To Peers...")
#print "block chain txhandler utxopool len " + str(len(self.node.block_chain.txhandler.utxopool.utxos))
#print "Node utxopool len " + str(len(self.node.utxopool.utxos))
#print "TX id is " + valid_txs[0].txid
self.node.utxopool = self.node.tx_handler.utxopool
self.node.wallet_manager.utxopool = self.node.utxopool
self.node.confirmed_utxopool = self.node.tx_handler.conf_utxopool
self.node.add_to_conf_utxopool(block)
chain_len = len(self.node.block_chain.chain)
size = os.path.getsize("{}_blockchain.bc".format(
self.node.node_name))
num_utxos = len(self.node.utxopool.utxos)
self.node.num_txs += len(block.txs)
self.node.Portal.update_blockchain_info(
chain_len, int(size), self.node.num_txs, num_utxos,
self.node.num_blk_mined)
self.node.Portal.update_blockchain_gui(block.index, block.hash)
self.node.wallet_manager.conf_utxopool = self.node.confirmed_utxopool
self.node.new_block_received = True
self.node.wallet_manager.update_utxopool_on_wallet()
self.node.wallet_manager.update_all_wallets_account_balance()
for t in block.txs:
try:
self.node.transaction_pool.remove(t)
except:
continue
reactor.callFromThread(self.node.send_block_via_clients, block)
else:
print("The Block Received is Owned By This Node!!")
def schedule(self, *args, **kwargs):
"""
Override in subclasses.
"""
reactor.callFromThread(*args, **kwargs)
bg.blit(instructions, (SCREEN_WIDTH-115, 0))
bg.blit(feed_pump_label, (80, 0))
bg.blit(oil_storage_label, (125, 100))
bg.blit(separator_label, (385,275))
screen.blit(waste_water_label, (265, 490))
bg.blit(tank_sensor, (125, 50))
bg.blit(outlet_sensor, (90, 195))
bg.blit(separator_release, (350, 375))
bg.blit(waste_sensor, (90, 375))
screen.blit(bg, (0, 0))
space.step(1/FPS)
pygame.display.flip()
if reactor.running:
reactor.callFromThread(reactor.stop)
store = ModbusSlaveContext(
di = ModbusSequentialDataBlock(0, [0]*100),
co = ModbusSequentialDataBlock(0, [0]*100),
hr = ModbusSequentialDataBlock(0, [0]*100),
ir = ModbusSequentialDataBlock(0, [0]*100))
context = ModbusServerContext(slaves=store, single=True)
# Modbus PLC server information
identity = ModbusDeviceIdentification()
identity.VendorName = 'Simmons Oil Refining Platform'
identity.ProductCode = 'SORP'
identity.VendorUrl = 'http://simmons.com/markets/oil-gas/pages/refining-industry.html'
identity.ProductName = 'SORP 3850'
def callback(indata, outdata, frames, time, status):
nonlocal jitter_buffer
nonlocal buf
nonlocal started
nonlocal sound_pos, sound_played
#print("in callback")
if status:
print(status)
# Input
# =====
# Apply automatic gain control, this will improve the
# quality of the audio that's sent.
#agc.apply(indata)
#print("gain:", agc.gain)
# Apply fixed gain
numpy.multiply(indata, gain, out=indata)
# Convert from float32 to int16
indata_int16 = indata * (2**15 - 1)
indata_int16 = indata_int16.astype(numpy.int16)
encoded_packets = opus_encoder.encode(indata_int16.tobytes())
# Send the encoded packets. Make sure not to call from
# this thread.
for encoded_packet in encoded_packets:
reactor.callFromThread(self._udp_packetizer.write,
encoded_packet)
# Check the discontinuous transmission (DTX) state
#print("DTX state:", opus_encoder.in_discontinuous_transmission())
# Output
# ======
def decode_next_packet():
nonlocal started
#print("in decode_next_packet")
encoded_packet = jitter_buffer.get_packet()
if encoded_packet is not None:
# Decode the encoded packet
pcm = opus_decoder.decode(encoded_packet)
started = True
else:
# Accept that we're missing the packet
if started:
print("WARNING Missing packet")
pcm = opus_decoder.decode_missing_packet(frame_size_ms)
else:
# We haven't even started, just output silence
#print("Haven't even started, return silence")
channels = outdata.shape[1]
samples = frame_size_ms * samples_per_second // 1000
pcm = numpy.zeros((samples, channels),
dtype=numpy.int16)
# Convert the data to floating point
pcm_int16 = numpy.frombuffer(pcm, dtype=numpy.int16)
pcm_float = pcm_int16.astype(numpy.float32)
pcm_float /= 2**15
pcm_float = numpy.reshape(pcm_float, (len(pcm_float), 1))
# DEBUG
if pcm_float.shape[
0] != frame_size_ms * samples_per_second // 1000:
print(
"FAIL Frame size isn't the desired duration ***********************************************"
)
print(f"It's first dimension is {pcm_float.shape[0]}.")
if pcm_float.shape[1] != 1: # channels
print(
"FAIL Frame size isn't the correct number of channels")
return pcm_float
# If there's insufficient data in buf attempt to obtain it
# from the jitter buffer
while buf is None or len(buf) < frames:
if buf is None:
buf = decode_next_packet()
else:
buf = numpy.concatenate((buf, decode_next_packet()))
# Copy the data from the buffer remove from the buffer than which
# we used.
outdata[:] = buf[:frames]
# This is INEFFICIENT and could be improved
buf = buf[frames:]
# If we haven't finished playing the sound, mix it in.
# TODO: This could be improved by fading in the audio signal
if not sound_played:
if sound_pos + frames < len(sound):
outdata[:] += sound[sound_pos:sound_pos + frames]
else:
outdata[:len(sound) -
sound_pos] += sound[sound_pos:len(sound)]
sound_played = True
sound_pos += frames
def wake():
time.sleep(0.1)
# callFromThread will call wakeUp for us
reactor.callFromThread(d.callback, None)
def tear_down(self):
"""
Tears down Twisted.
"""
reactor.callFromThread(reactor.stop)
self._reactor_thread.join()
def onExitSignal(*a):
print 'Closing Timeline?'
if not reactor.running:
os._exit(1)
reactor.callFromThread(reactor.stop)
def stopThreadedServer():
reactor.callFromThread(reactor.stop)
def _insert(self, item, col, out):
self.mongodb[col].insert_one(dict(item))
reactor.callFromThread(out.callback, item)
def request_message(self, message):
reactor.callFromThread(self.factory.protocol.reqeuset_message, message)
def run_sighup(*args, **kwargs):
# `callFromThread` should be "signal safe" as well as thread
# safe.
reactor.callFromThread(handle_sighup, *args, **kwargs)
def _signal_kill(self, signum, _):
install_shutdown_handlers(signal.SIG_IGN)
signame = signal_names[signum]
log.msg(format='Received %(signame)s twice, forcing unclean shutdown',
level=log.INFO, signame=signame)
reactor.callFromThread(self._stop_reactor)
def close(self):
reactor.callFromThread(self.factory.protocol.close)
def _signal_shutdown(self, signum, _):
install_shutdown_handlers(self._signal_kill)
signame = signal_names[signum]
log.msg(format="Received %(signame)s, shutting down gracefully. Send again to force ",
level=log.INFO, signame=signame)
reactor.callFromThread(self.stop)
def decorator(*args, **kwargs):
reactor.callFromThread(func, *args, **kwargs)