class LeakyBucketProcessor(object):
"""
Обработчик очереди задач по алгоритму https://en.wikipedia.org/wiki/Leaky_bucket
Размер очереди определяется параметром bandwidth при создании класса
"""
def __init__(self, bandwidth=5):
"""
:param bandwidth: максимальная длина очереди, по превышению которой запросы будут отсекаться
"""
self.tasks = []
self.bandwidth = bandwidth
self._lock = DeferredLock()
def do_next(self):
"""запуск очередной итерации, вызов идет блокирующий"""
self._lock.run(self._do_next)
def _do_next(self):
# обработка задач в очереди
if self.tasks:
task = self.tasks.pop(0)
# запускаем коллбэк в отдельном thread
reactor.callInThread(task['callback'],
message="task done: {}".format(task['data']),
task=task)
def add_task(self, data, callback):
"""
добавление задачи в очередь, если есть место, вызов идет блокирующий
:param data: данные для задачи
:param callback: функция которая будет вызвана по завершению обработки задачи
"""
return self._lock.run(self._add_task, {
"data": data,
"callback": callback
})
def _add_task(self, task):
# добавление задачи в очередь
# если нет переполнения очереди, то добавляем задачу
if len(self.tasks) < self.bandwidth:
self.tasks.append(task)
# запускаем коллбэк в отдельном thread
reactor.callInThread(task['callback'],
message="task added: {}".format(task['data']),
task=task)
else:
# запускаем коллбэк в отдельном thread
reactor.callInThread(task['callback'],
message="task refused: {}".format(
task['data']),
task=task)
class MKSPDR2000Wrapper(DeviceWrapper):
@inlineCallbacks
def connect(self, server, port):
'''Connect to the MKS PDR2000'''
print('Connecting to "%s" on port "%s"...' % (server.name, port))
self.server = server
self.ctx = server.context()
self.port = port
p = self.packet()
p.open(port)
# The following parameters were obtained from the MKS PDR2000 manual.
p.baudrate(9600L)
p.stopbits(1L)
p.bytesize(8L)
p.parity('N')
p.timeout(0.1 * u.s)
# Clear out the Rx buffer. This is necessary for some devices.
yield p.send()
def packet(self):
'''Create a new packet in our private context'''
return self.server.packet(context=self.ctx)
def shutdown(self):
'''Disconnect from teh serial port when we shut down.'''
return self.packet().close().send()
def rw_line(self, code):
# Don't allow two concurrent read/write calls.
self._lock = DeferredLock()
return self._lock.run(partial(self._rw_line, code))
@inlineCallbacks
def _rw_line(self, code):
'''Write data to the device.'''
yield self.server.write_line(code, context=self.ctx)
time.sleep(0.2)
ans = yield self.server.read(context=self.ctx)
returnValue(ans)
@inlineCallbacks
def getUnits(self):
yield self.write_line('u')
time.sleep(0.5)
ans = yield self.read_line()
returnValue(ans)
def toFloat(self, val):
try:
if val == 'Off' or val == 'Low':
return np.nan
return float(val)
except:
return None
class MKSPDR2000Wrapper(DeviceWrapper):
@inlineCallbacks
def connect(self, server, port):
'''Connect to the MKS PDR2000'''
print('Connecting to "%s" on port "%s"...' %(server.name, port))
self.server = server
self.ctx = server.context()
self.port = port
p = self.packet()
p.open(port)
# The following parameters were obtained from the MKS PDR2000 manual.
p.baudrate(9600L)
p.stopbits(1L)
p.bytesize(8L)
p.parity('N')
p.timeout(0.1 * u.s)
# Clear out the Rx buffer. This is necessary for some devices.
yield p.send()
def packet(self):
'''Create a new packet in our private context'''
return self.server.packet(context=self.ctx)
def shutdown(self):
'''Disconnect from teh serial port when we shut down.'''
return self.packet().close().send()
def rw_line(self, code):
# Don't allow two concurrent read/write calls.
self._lock = DeferredLock()
return self._lock.run(partial(self._rw_line, code))
@inlineCallbacks
def _rw_line(self, code):
'''Write data to the device.'''
yield self.server.write_line(code, context=self.ctx)
time.sleep(0.2)
ans = yield self.server.read(context=self.ctx)
returnValue(ans)
@inlineCallbacks
def getUnits(self):
yield self.write_line('u')
time.sleep(0.5)
ans = yield self.read_line()
returnValue(ans)
def toFloat(self, val):
try:
if val == 'Off' or val == 'Low':
return np.nan
return float(val)
except:
return None
class BNCSerial(BNCPulser):
@inlineCallbacks
def connect(self, server, port, baud):
self.server = server
#usage ensures only one read/write operation at a time
self._portLock = DeferredLock()
print 'connecting to "%s" on port "%s"...' % (server.name, port)
p = self.server.packet()
p.open(port)
p.baudrate(baud)
p.parity('N') #no parity bit
p.bytesize(8L)
p.stopbits(1L)
p.write_line(':SYST:COMM:SER:ECHO ON')
yield p.send()
chList = yield self.channel_list()
self.chMap = dict(chList)
@inlineCallbacks
def query(self, line):
yield self._portLock.acquire()
p = self.server.packet()
resp = yield p.read()\
.write_line(line)\
.pause(U.Value(40.0, 'ms'))\
.read_line()\
.send()
print 'RESPONSE: %s' % resp['read_line']
returnValue(resp['read_line'])
@inlineCallbacks
def send(self, line):
yield self._portLock.run(self.server.write_line, line)
@inlineCallbacks
def all_ch_states(self):
chns = self.chMap.keys()
p = self.server.packet()
p.read() #clear buffer
for n in chns:
p.write(':PULSE%d:STATE?' % n)
p.pause(U.Value(30, 'ms'))
p.read(key=str(n))
resp = yield p.send()
parser = TYPE_MAP['bool'][1]
returnValue([(n, parser(resp[str(n)])) for n in chns])
class VarianControllerWrapper(DeviceWrapper):
@inlineCallbacks
def connect(self, server, port):
'''Connect the the guage controller'''
print('Connecting to "%s" on port "%s"...' %(server.name, port))
self.server = server
self.ctx = server.context()
self.port = port
# The following parameters match the default configuration of
# the Varian unit.
p = self.packet()
p.open(port)
p.baudrate(9600L)
p.stopbits(1L)
p.bytesize(8L)
p.parity('N')
p.rts(False)
p.timeout(2 * units.s)
# Clear out the read buffer. This is necessary for some devices.
p.read_line()
yield p.send()
def packet(self):
"""Create a packet in our private context."""
return self.server.packet(context=self.ctx)
def shutdown(self):
"""Disconnect from the serial port when we shut down."""
return self.packet().close().send()
def rw_line(self, code):
# Don't allow two concurrent read/write calls. Use deferred locking to
# enforce this
self._lock = DeferredLock()
return self._lock.run(partial(self._rw_line, code))
@inlineCallbacks
def _rw_line(self, code):
'''Write data to the device.'''
yield self.server.write_line(code, context=self.ctx)
time.sleep(0.1)
ans = yield self.server.read(context=self.ctx)
returnValue(ans)
class VarianControllerWrapper(DeviceWrapper):
@inlineCallbacks
def connect(self, server, port):
"""Connect the the guage controller."""
print('Connecting to "%s" on port "%s"...' %(server.name, port))
self.server = server
self.ctx = server.context()
self.port = port
# The following parameters match the default configuration of
# the Varian unit.
p = self.packet()
p.open(port)
p.baudrate(9600L)
p.stopbits(1L)
p.bytesize(8L)
p.parity('N')
p.rts(False)
p.timeout(2 * units.s)
# Clear out the read buffer. This is necessary for some devices.
p.read_line()
yield p.send()
def packet(self):
"""Create a packet in our private context."""
return self.server.packet(context=self.ctx)
def shutdown(self):
"""Disconnect from the serial port when we shut down."""
return self.packet().close().send()
def rw_line(self, code):
# Don't allow two concurrent read/write calls. Use deferred locking to
# enforce this
self._lock = DeferredLock()
return self._lock.run(partial(self._rw_line, code))
@inlineCallbacks
def _rw_line(self, code):
"""Write data to the device."""
yield self.server.write_line(code, context=self.ctx)
yield sleep(0.1)
ans = yield self.server.read(context=self.ctx)
returnValue(ans)
class BrowserMiddleware(object):
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
if crawler.settings.getbool('BROWSER_ENGINE_COOKIES_ENABLED', False):
if crawler.settings.getbool('COOKIES_ENABLED'):
logger.warning("Default cookies middleware enabled together "
"with browser engine aware cookies middleware. "
"Set COOKIES_ENABLED to False.")
cookies_mw = RemotelyAccessbileCookiesMiddleware(
debug=crawler.settings.getbool('COOKIES_DEBUG'))
else:
cookies_mw = None
server = settings.get('BROWSER_ENGINE_SERVER')
if server:
endpoint = clientFromString(reactor, server)
else:
if settings.getbool('BROWSER_ENGINE_START_SERVER', False):
# Twisted logs the process's stderr with INFO level.
logging.getLogger("twisted").setLevel(logging.INFO)
argv = [
sys.executable, "-m", "scrapy_qtwebkit.browser_engine",
"stdio"
]
endpoint = ProcessEndpoint(reactor, argv[0], argv, env=None)
else:
raise NotConfigured(
"Must provide either BROWSER_ENGINE_SERVER "
"or BROWSER_ENGINE_START_SERVER")
ext = cls(
crawler,
endpoint,
page_limit=settings.getint('BROWSER_ENGINE_PAGE_LIMIT', 4),
cookies_middleware=cookies_mw,
)
return ext
def __init__(self,
crawler,
client_endpoint,
page_limit=4,
cookies_middleware=None):
super().__init__()
self._crawler = crawler
self._client_endpoint = client_endpoint
self.page_limit = page_limit
self.cookies_mw = cookies_middleware
self._downloader = None
self._browser = None
self._browser_init_lock = DeferredLock()
@inlineCallbacks
def _init_browser(self):
# XXX: open at most one browser at a time per client (i.e. per Scrapy
# instance), ensure there is no communication between pages of a
# browser. If not possible, open one browser per cookiejar but also
# allow the user to have separate browsers on the same cookiejar.
if self._browser is not None:
return
factory = pb.PBClientFactory(security=jelly.DummySecurityOptions())
broker = yield self._client_endpoint.connect(factory)
if isinstance(self._client_endpoint, ProcessEndpoint):
atexit.register(broker.transport.signalProcess, "TERM")
# Endpoints do not call ClientFactory.clientConnectionLost(), so the
# factory never calls back its own getRootObject() deferreds when an
# error happens.
root_dfd = factory.getRootObject()
broker.notifyOnDisconnect(partial(root_dfd.errback, ConnectionLost()))
root = yield root_dfd
if self._downloader is None:
self._downloader = BrowserRequestDownloader(self._crawler)
self._browser = yield root.callRemote('open_browser', self._downloader)
@inlineCallbacks
def _get_browser(self):
if self._browser is None:
yield self._browser_init_lock.run(self._init_browser)
return self._browser
@inlineCallbacks
def process_request(self, request, spider):
if self.cookies_mw:
yield self.cookies_mw.process_request(request, spider)
if isinstance(request, BrowserRequest):
response_dfd = self._make_browser_request(request, spider)
return (yield response_dfd)
def process_response(self, request, response, spider):
if self.cookies_mw:
return self.cookies_mw.process_response(request, response, spider)
else:
return response
@inlineCallbacks
def _make_browser_request(self, request, spider):
browser = yield self._get_browser()
webpage_options = {
'count_increaser': _RequestCountRemoteIncreaser(request),
'user_agent': request.headers.get('User-Agent')
}
if self.cookies_mw and 'dont_merge_cookies' not in request.meta:
cookiejarkey = request.meta.get("cookiejar")
cookiejar = self.cookies_mw.jars[cookiejarkey].jar
webpage_options['cookiejarkey'] = cookiejarkey
webpage_options['cookiejar'] = cookiejar
webpage = yield browser.callRemote('create_webpage', webpage_options)
result = webpage.callRemote(
'load_request',
RequestFromScrapy(request.url, request.method,
dict(request.headers), request.body))
result.addCallback(partial(self._handle_page_load, request, webpage))
return (yield result)
@inlineCallbacks
def _handle_page_load(self,
request,
webpage,
load_result=(True, 200, None, None)):
"""
Handle a request for a web page, either a page load or a request to
continue using an existing page object.
"""
try:
ok, status, headers, exc = load_result
if ok:
url = yield webpage.callRemote('get_url')
browser_response = request.meta.get('browser_response', False)
if browser_response:
respcls = BrowserResponse
else:
respcls = HtmlResponse
encoding, body = yield webpage.callRemote('get_body')
response = respcls(status=status,
url=url,
headers=headers,
body=body,
encoding=encoding,
request=request)
if browser_response:
response.webpage = PbReferenceMethodsWrapper(webpage)
else:
if isinstance(exc, ScrapyNotSupported):
exc = NotSupported(*exc.args)
raise exc
except Exception as err:
response = Failure(err)
return response
class HendrixDeploy(object):
"""
HendrixDeploy encapsulates the necessary information needed to deploy
the HendrixService on a single or multiple processes.
"""
def __init__(self, action='start', options={},
reactor=reactor, threadpool=None):
self.action = action
self.options = hx_options()
self.options.update(options)
self.services = []
self.resources = []
self.reactor = reactor
self.threadpool = threadpool or ThreadPool(name="Hendrix Web Service")
self.use_settings = True
# because running the management command overrides self.options['wsgi']
if self.options['wsgi']:
if hasattr(self.options['wsgi'], '__call__'):
# If it has a __call__, we assume that it is the application
# object itself.
self.application = self.options['wsgi']
try:
self.options['wsgi'] = "%s.%s" % (
self.application.__module__, self.application.__name__
)
except AttributeError:
self.options['wsgi'] = self.application.__class__.__name__
else:
# Otherwise, we'll try to discern an application in the belief
# that this is a dot path.
wsgi_dot_path = self.options['wsgi']
# will raise AttributeError if we can't import it.
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.use_settings = False
else:
os.environ['DJANGO_SETTINGS_MODULE'] = self.options['settings']
settings = import_string('django.conf.settings')
self.services = get_additional_services(settings)
self.resources = get_additional_resources(settings)
self.options = HendrixDeploy.getConf(settings, self.options)
if self.use_settings:
django = importlib.import_module('django')
if django.VERSION[:2] >= (1, 7):
django.setup()
wsgi_dot_path = getattr(settings, 'WSGI_APPLICATION', None)
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.is_secure = self.options['key'] and self.options['cert']
self.servers = []
self._lock = DeferredLock()
@classmethod
def importWSGI(cls, wsgi_dot_path):
try:
wsgi_module, application_name = wsgi_dot_path.rsplit('.', 1)
except AttributeError:
pid = os.getpid()
chalk.red(
"Unable to discern a WSGI application from '%s'" %
wsgi_dot_path
)
os.kill(pid, 15)
try:
wsgi = importlib.import_module(wsgi_module)
except ImportError:
chalk.red("Unable to Import module '%s'\n" % wsgi_dot_path)
raise
return getattr(wsgi, application_name, None)
@classmethod
def getConf(cls, settings, options):
"updates the options dict to use config options in the settings module"
ports = ['http_port', 'https_port', 'cache_port']
for port_name in ports:
port = getattr(settings, port_name.upper(), None)
# only use the settings ports if the defaults were left unchanged
default = getattr(defaults, port_name.upper())
if port and options.get(port_name) == default:
options[port_name] = port
_opts = [
('key', 'hx_private_key'),
('cert', 'hx_certficate'),
('wsgi', 'wsgi_application')
]
for opt_name, settings_name in _opts:
opt = getattr(settings, settings_name.upper(), None)
if opt:
options[opt_name] = opt
if not options['settings']:
options['settings'] = environ['DJANGO_SETTINGS_MODULE']
return options
def addServices(self):
"""
a helper function used in HendrixDeploy.run
it instanstiates the HendrixService and adds child services
note that these services will also be run on all processes
"""
self.addHendrix()
def addGlobalServices(self):
"""
This is where we put service that we don't want to be duplicated on
worker subprocesses
"""
pass
def getThreadPool(self):
'''
Case to match twisted.internet.reactor
'''
return self.threadpool
def addHendrix(self):
'''
Instantiates a HendrixService with this object's threadpool.
It will be added as a service later.
'''
self.hendrix = HendrixService(
self.application,
threadpool=self.getThreadPool(),
resources=self.resources,
services=self.services,
loud=self.options['loud']
)
if self.options["https_only"] is not True:
self.hendrix.spawn_new_server(self.options['http_port'], HendrixTCPService)
def catalogServers(self, hendrix):
"collects a list of service names serving on TCP or SSL"
for service in hendrix.services:
if isinstance(service, (TCPServer, SSLServer)):
self.servers.append(service.name)
def _listening_message(self):
message = "non-TLS listening on port {}".format(self.options['http_port'])
return message
def run(self):
"sets up the desired services and runs the requested action"
self.addServices()
self.catalogServers(self.hendrix)
action = self.action
fd = self.options['fd']
if action.startswith('start'):
chalk.blue(self._listening_message())
getattr(self, action)(fd)
###########################
# annnnd run the reactor! #
###########################
try:
self.reactor.run()
finally:
shutil.rmtree(PID_DIR, ignore_errors=True) # cleanup tmp PID dir
elif action == 'restart':
getattr(self, action)(fd=fd)
else:
getattr(self, action)()
@property
def pid(self):
"The default location of the pid file for process management"
return get_pid(self.options)
def getSpawnArgs(self):
_args = [
'hx',
'start', # action
# kwargs
'--http_port', str(self.options['http_port']),
'--https_port', str(self.options['https_port']),
'--cache_port', str(self.options['cache_port']),
'--workers', '0',
'--fd', pickle.dumps(self.fds),
]
# args/signals
if self.options['dev']:
_args.append('--dev')
if not self.use_settings:
_args += ['--wsgi', self.options['wsgi']]
return _args
def start(self, fd=None):
if fd is None:
# anything in this block is only run once
self.addGlobalServices()
self.hendrix.startService()
pids = [str(os.getpid())] # script pid
if self.options['workers']:
self.launchWorkers(pids)
self.pid_file = self.openPidList(pids)
else:
fds = pickle.loads(fd)
factories = {}
for name in self.servers:
factory = self.disownService(name)
factories[name] = factory
self.hendrix.startService()
for name, factory in factories.iteritems():
self.addSubprocess(fds, name, factory)
chalk.eraser()
chalk.blue('Starting Hendrix...')
def setFDs(self):
"""
Iterator for file descriptors.
Seperated from launchworkers for clarity and readability.
"""
# 0 corresponds to stdin, 1 to stdout, 2 to stderr
self.childFDs = {0: 0, 1: 1, 2: 2}
self.fds = {}
for name in self.servers:
self.port = self.hendrix.get_port(name)
fd = self.port.fileno()
self.childFDs[fd] = fd
self.fds[name] = fd
def launchWorkers(self, pids):
# Create a new listening port and several other processes to
# help out.
self.setFDs()
args = self.getSpawnArgs()
transports = []
for i in range(self.options['workers']):
time.sleep(0.05)
transport = self.reactor.spawnProcess(
DeployServerProtocol(args), 'hx', args, childFDs=self.childFDs, env=environ
)
transports.append(transport)
pids.append(str(transport.pid))
def openPidList(self, pids):
with open(self.pid, 'w') as pid_file:
pid_file.write('\n'.join(pids))
return pid_file
def addSubprocess(self, fds, name, factory):
"""
Public method for _addSubprocess.
Wraps reactor.adoptStreamConnection in
a simple DeferredLock to guarantee
workers play well together.
"""
self._lock.run(self._addSubprocess, self, fds, name, factory)
def _addSubprocess(self, fds, name, factory):
self.reactor.adoptStreamConnection(
fds[name], AF_INET, factory
)
def stop(self, sig=9):
with open(self.pid) as pid_file:
pids = pid_file.readlines()
for pid in pids:
try:
os.kill(int(pid), sig)
except OSError:
# OSError raised when it trys to kill the child processes
pass
os.remove(self.pid)
chalk.green('Stopping Hendrix...')
def start_reload(self, fd=None):
self.start(fd=fd)
def restart(self, fd=None):
self.stop()
time.sleep(1) # wait a second to ensure the port is closed
self.start(fd)
def disownService(self, name):
"""
disowns a service on hendirix by name
returns a factory for use in the adoptStreamPort part of setting up
multiple processes
"""
_service = self.hendrix.getServiceNamed(name)
_service.disownServiceParent()
return _service.factory
def add_non_tls_websocket_service(self, websocket_factory):
autobahn.twisted.websocket.listenWS(websocket_factory)
class WorkQueue(object):
"""A WorkQueue contains WorkUnits and dispatches NonceRanges when requested
by the miner. WorkQueues dispatch deffereds when they runs out of nonces.
"""
def __init__(self, core):
self.core = core
self.logger = core.logger
self.queueSize = core.config.get('general', 'queuesize', int, 1)
self.queueDelay = core.config.get('general', 'queuedelay', int, 5)
self.lock = DeferredLock()
self.queue = deque('', self.queueSize)
self.deferredQueue = deque()
self.currentUnit = None
self.lastBlock = None
self.block = ''
self.staleCallbacks = []
def storeWork(self, aw):
#check if this work matches the previous block
if (self.lastBlock is not None) and (aw.identifier == self.lastBlock):
self.logger.debug('Server gave work from the previous '
'block, ignoring.')
#if the queue is too short request more work
if self.checkQueue():
if self.core.connection:
self.core.connection.requestWork()
return
#create a WorkUnit
work = WorkUnit(aw)
reactor.callLater(
max(60, aw.time - 1) - self.queueDelay, self.checkWork)
reactor.callLater(max(60, aw.time - 1), self.workExpire, work)
#check if there is a new block, if so reset queue
newBlock = (aw.identifier != self.block)
if newBlock:
self.queue.clear()
self.currentUnit = None
self.lastBlock = self.block
self.block = aw.identifier
self.logger.debug("New block (WorkQueue)")
#add new WorkUnit to queue
if work.data and work.target and work.midstate and work.nonces:
self.queue.append(work)
#if the queue is too short request more work
workRequested = False
if self.checkQueue():
if self.core.connection:
self.core.connection.requestWork()
workRequested = True
#if there is a new block notify kernels that their work is now stale
if newBlock:
for callback in self.staleCallbacks:
callback()
self.staleCallbacks = []
self.staleCallbacks.append(work.stale)
#check if there are deferred WorkUnit requests pending
#since requests to fetch a WorkUnit can add additional deferreds to
#the queue, cache the size beforehand to avoid infinite loops.
for i in range(len(self.deferredQueue)):
df = self.deferredQueue.popleft()
d = self.fetchUnit(workRequested)
d.chainDeferred(df)
#clear the idle flag since we just added work to queue
self.core.reportIdle(False)
def checkWork(self):
# Called 5 seconds before any work expires in order to fetch more
if self.checkQueue():
if self.core.connection:
self.core.requestWork()
def checkQueue(self, added=False):
# This function checks the queue length including the current unit
size = 1
# Check if the current unit will last long enough
if self.currentUnit is None:
if len(self.queue) == 0:
return True
else:
size = 0
if added:
rolls = self.queue[0].maxtime - self.queue[0].timestamp
# If new work can't be rolled, and queue would be too small
if rolls == 0 and (len(self.queue) - 1) < self.queueSize:
return True
else:
remaining = self.currentUnit.maxtime - self.currentUnit.timestamp
# Check if we are about to run out of rolltime on current unit
if remaining < (self.queueDelay):
size = 0
# Check if the current unit is about to expire
age = self.currentUnit.downloaded + self.currentUnit.time
lifetime = age - time()
if lifetime < (2 * self.queueDelay):
size = 0
# Check if the queue will last long enough
queueLength = 0
for i in range(len(self.queue)):
age = self.queue[0].downloaded + max(60, self.queue[0].time - 1)
lifetime = age - time()
if lifetime > (2 * self.queueDelay):
queueLength += 1
# Return True/False indicating if more work should be fetched
return size + queueLength < self.queueSize
def workExpire(self, wu):
# Don't expire WorkUnits if idle and queue empty
if (self.core.idle) and (len(self.queue) <= 1):
return
# Remove the WorkUnit from queue
if len(self.queue) > 0:
iSize = len(self.queue)
if not (len(self.queue) == 1 and (self.currentUnit is None)):
try:
self.queue.remove(wu)
except ValueError:
pass
if self.currentUnit == wu:
self.currentUnit = None
# Check queue size
if self.checkQueue() and (iSize != len(self.queue)):
if self.core.connection:
self.core.connection.requestWork()
# Flag the WorkUnit as stale
wu.stale()
else:
# Check back again later if we didn't expire the work
reactor.callLater(5, self.workExpire, wu)
def getRangeFromUnit(self, size):
#get remaining nonces
noncesLeft = self.currentUnit.nonces - self.currentUnit.base
# Flag indicating if the WorkUnit was depeleted by this request
depleted = False
#if there are enough nonces to fill the full reqest
if noncesLeft >= size:
nr = NonceRange(self.currentUnit, self.currentUnit.base, size)
#check if this uses up the rest of the WorkUnit
if size >= noncesLeft:
depleted = True
else:
self.currentUnit.base += size
#otherwise send whatever is left
else:
nr = NonceRange(self.currentUnit, self.currentUnit.base,
noncesLeft)
depleted = True
#return the range
return nr, depleted
def checkRollTime(self, wu):
# This function checks if a WorkUnit could be time rolled
if wu.maxtime > wu.timestamp and not wu.isStale:
remaining = (wu.downloaded + wu.time) - time()
if remaining > (self.queueDelay) or len(self.queue) < 1:
# If it has been more than 5 minutes probably better to idle
if time() - wu.downloaded < 300:
return True
return False
def rollTime(self, wu):
# Check if this WorkUnit supports rolling time, return None if not
if not self.checkRollTime(wu):
return None
# Create the new WU
newWU = WorkUnit(wu)
# Increment the timestamp
newWU.timestamp += 1
# Reset the download time to the original WU's
newWU.downloaded = wu.downloaded
# Set a stale callback for this WU
self.staleCallbacks.append(newWU.stale)
# Setup a workExpire callback
remaining = max(self.queueDelay, (wu.downloaded + wu.time) - time())
reactor.callLater(remaining - 1, self.workExpire, newWU)
# Return the new WU
return newWU
def fetchUnit(self, delayed=False):
#if there is a unit in queue
if len(self.queue) >= 1:
#check if the queue has fallen below the desired size
if self.checkQueue(True) and (not delayed):
#Request more work to maintain minimum queue size
if self.core.connection:
self.core.connection.requestWork()
#get the next unit from queue
wu = self.queue.popleft()
#return the unit
return defer.succeed(wu)
#if the queue is empty
else:
#request more work
if self.core.connection:
self.core.connection.requestWork()
#report that the miner is idle
self.core.reportIdle(True)
#set up and return deferred
df = defer.Deferred()
self.deferredQueue.append(df)
return df
#make sure that only one fetchRange request runs at a time
def fetchRange(self, size=0x10000):
return self.lock.run(self._fetchRange, size)
def _fetchRange(self, size):
#make sure size is not too large
size = min(size, 0x100000000)
#check if the current unit exists
if self.currentUnit is not None:
# Get a nonce range
nr, depleated = self.getRangeFromUnit(size)
# If we depleted the Workunit then try to roll time
if depleated:
self.currentUnit = self.rollTime(self.currentUnit)
# Return the range
return defer.succeed(nr)
#if there is no current unit
else:
# Check if we can get a new unit with rolltime
def callback(wu):
#get a new current unit
self.currentUnit = wu
#get a nonce range
nr, depleated = self.getRangeFromUnit(size)
# If we depleted the Workunit then try to roll time
if depleated:
self.currentUnit = self.rollTime(self.currentUnit)
#return the range
return nr
d = self.fetchUnit()
d.addCallback(callback)
return d
class BrowserMiddleware(object):
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
if crawler.settings.getbool('BROWSER_ENGINE_COOKIES_ENABLED', False):
if crawler.settings.getbool('COOKIES_ENABLED'):
logger.warning("Default cookies middleware enabled together "
"with browser engine aware cookies middleware. "
"Set COOKIES_ENABLED to False.")
cookies_mw = RemotelyAccessibleCookiesMiddleware(
debug=crawler.settings.getbool('COOKIES_DEBUG'))
else:
cookies_mw = None
server = settings.get('BROWSER_ENGINE_SERVER')
start_server = settings.getbool('BROWSER_ENGINE_START_SERVER', False)
if not (server or start_server):
raise NotConfigured("Must specify either BROWSER_ENGINE_SERVER or "
"BROWSER_ENGINE_START_SERVER")
if server and start_server:
raise NotConfigured("Must not specify both BROWSER_ENGINE_SERVER "
"and BROWSER_ENGINE_START_SERVER=True")
if server:
endpoint = clientFromString(reactor, server)
else:
# Twisted logs the process's stderr with INFO level.
logging.getLogger("twisted").setLevel(logging.INFO)
argv = [
sys.executable, "-m", "scrapy_qtwebkit.browser_engine", "stdio"
]
endpoint = ProcessEndpoint(reactor, argv[0], argv, env=None)
mw = cls(
crawler,
endpoint,
page_limit=settings.getint('BROWSER_ENGINE_PAGE_LIMIT', 4),
browser_options=settings.getdict('BROWSER_ENGINE_OPTIONS'),
cookies_middleware=cookies_mw,
)
crawler.signals.connect(mw._engine_stopped,
signal=signals.engine_stopped)
return mw
def __init__(self,
crawler,
client_endpoint,
page_limit=4,
browser_options=None,
cookies_middleware=None):
super().__init__()
self._crawler = crawler
self._client_endpoint = client_endpoint
if page_limit:
self._semaphore = DeferredSemaphore(page_limit)
else:
self._semaphore = DummySemaphore()
self.browser_options = (browser_options or {})
self.cookies_mw = cookies_middleware
self._downloader = BrowserRequestDownloader(self._crawler)
self._browser = None
self._browser_init_lock = DeferredLock()
@inlineCallbacks
def _init_browser(self):
# XXX: open at most one browser at a time per client (i.e. per Scrapy
# instance), ensure there is no communication between pages of a
# browser. If not possible, open one browser per cookiejar but also
# allow the user to have separate browsers on the same cookiejar.
if self._browser is not None:
return
# The endpoint does not call the factory's clientConnectionLost()
# method. PBClientFactory relies on this method being called in order
# to fail pending getRootObject() requests, thus a failed connection
# would not cause the failure of getRootObject(), which would hang
# forever.
# The endpoint does call the protocol's connectionLost(), so a protocol
# that calls the factory's clientConnectionLost() seems to solve this
# problem.
factory = pb.PBClientFactory(security=jelly.DummySecurityOptions())
factory.protocol = PBBrokerForEndpoint
yield self._client_endpoint.connect(factory)
root = yield factory.getRootObject()
self._browser = yield root.callRemote('open_browser',
downloader=self._downloader,
options=self.browser_options)
def _engine_stopped(self):
# Must run after BrowserResponseTrackerMiddleware._spider_closed().
try:
self._browser.broker.transport.signalProcess("TERM")
except AttributeError:
pass
@inlineCallbacks
def _get_browser(self):
if self._browser is None:
yield self._browser_init_lock.run(self._init_browser)
return self._browser
@inlineCallbacks
def process_request(self, request, spider):
if self.cookies_mw:
yield self.cookies_mw.process_request(request, spider)
if isinstance(request, BrowserRequest):
response = yield self._make_browser_request(request)
return response
def process_response(self, request, response, spider):
if self.cookies_mw:
return self.cookies_mw.process_response(request, response, spider)
else:
return response
@inlineCallbacks
def _make_browser_request(self, request):
browser = yield self._get_browser()
options = {
'remote_request_counter': request.remote_counter,
'user_agent': request.headers.get('User-Agent')
}
if self.cookies_mw and 'dont_merge_cookies' not in request.meta:
cookiejarkey = request.meta.get("cookiejar")
cookiejar = self.cookies_mw.jars[cookiejarkey].jar
options['cookiejarkey'] = cookiejarkey
options['cookiejar'] = cookiejar
else:
cookiejar = None
yield self._semaphore.acquire()
try:
webpage = yield browser.callRemote('create_webpage', options)
except:
self._semaphore.release()
raise
if cookiejar:
yield cookiejar.sync()
yield webpage.callRemote('_commit_cookies')
result = webpage.callRemote(
'load_request',
RequestFromScrapy(request.url, request.method, request.headers,
request.body))
result.addCallback(
partial(self._handle_page_load, request, webpage, cookiejar))
del webpage
return (yield result)
@inlineCallbacks
def _handle_page_load(self, request, webpage, cookiejar, load_result):
if cookiejar:
yield sync_cookies(cookiejar, webpage)
browser_response = request.meta.get('browser_response', False)
try:
ok, status, headers, exc = load_result
if ok:
if browser_response:
respcls = BrowserResponse
else:
respcls = HtmlResponse
url = yield webpage.callRemote('get_url')
encoding, body = yield webpage.callRemote('get_body')
response = respcls(status=status,
url=url,
headers=headers,
body=body,
encoding=encoding,
request=request)
if browser_response:
response._webpage = PBReferenceMethodsWrapper(webpage)
response._semaphore = self._semaphore
response._cookiejar = cookiejar
else:
if isinstance(exc, ScrapyNotSupported):
exc = NotSupported(*exc.args)
raise exc
except Exception as err:
browser_response = False
response = Failure(err)
finally:
if not browser_response:
try:
yield webpage.callRemote('close')
finally:
self._semaphore.release()
return response
class GPIBDeviceManager(LabradServer):
"""Manages autodetection and identification of GPIB devices.
The device manager listens for "GPIB Device Connect" and
"GPIB Device Disconnect" messages coming from GPIB bus servers.
It attempts to identify the connected devices and forward the
messages on to servers interested in particular devices. For
devices that cannot be identified by *IDN? in the usual way,
servers can register an identification setting to be called
by the device manager to properly identify the device.
"""
name = 'GPIB Device Manager'
@inlineCallbacks
def initServer(self):
"""Initialize the server after connecting to LabRAD."""
self.knownDevices = {} # maps (server, channel) to (name, idn)
self.deviceServers = {} # maps device name to list of interested servers
self.identFunctions = {} # maps server to (setting, ctx) for ident
self.identLock = DeferredLock()
# named messages are sent with source ID first, which we ignore
connect_func = lambda c, (s, payload): self.gpib_device_connect(*payload)
disconnect_func = lambda c, (s, payload): self.gpib_device_disconnect(*payload)
mgr = self.client.manager
self._cxn.addListener(connect_func, source=mgr.ID, ID=10)
self._cxn.addListener(disconnect_func, source=mgr.ID, ID=11)
yield mgr.subscribe_to_named_message('GPIB Device Connect', 10, True)
yield mgr.subscribe_to_named_message('GPIB Device Disconnect', 11, True)
# do an initial scan of the available GPIB devices
yield self.refreshDeviceLists()
@inlineCallbacks
def refreshDeviceLists(self):
"""Ask all GPIB bus servers for their available GPIB devices."""
servers = [s for n, s in self.client.servers.items()
if (('GPIB Bus' in n) or ('gpib_bus' in n) or ('sim900' in n) or ('SIM900' in n)) and \
(('List Addresses' in s.settings) or \
('list_addresses' in s.settings))]
names = [s.name for s in servers]
print 'Pinging servers:', names
resp = yield DeferredList([s.list_addresses() for s in servers])
for name, (success, addrs) in zip(names, resp):
if not success:
print 'Failed to get device list for:', name
else:
print 'Server %s has devices: %s' % (name, addrs)
for addr in addrs:
self.gpib_device_connect(name, addr)
@inlineCallbacks
def gpib_device_connect(self, server, channel):
"""Handle messages when devices connect."""
print 'Device Connect:', server, channel
if (server, channel) in self.knownDevices:
return
device, idnResult = yield self.lookupDeviceName(server, channel)
if device == UNKNOWN:
device = yield self.identifyDevice(server, channel, idnResult)
self.knownDevices[server, channel] = (device, idnResult)
# forward message if someone cares about this device
if device in self.deviceServers:
self.notifyServers(device, server, channel, True)
def gpib_device_disconnect(self, server, channel):
"""Handle messages when devices connect."""
print 'Device Disconnect:', server, channel
if (server, channel) not in self.knownDevices:
return
device, idnResult = self.knownDevices[server, channel]
del self.knownDevices[server, channel]
# forward message if someone cares about this device
if device in self.deviceServers:
self.notifyServers(device, server, channel, False)
@inlineCallbacks
def lookupDeviceName(self, server, channel):
"""Try to send a *IDN? or an alternative query to lookup info about a device.
Returns the name of the device and the actual response string
to the identification query. If the response cannot be parsed
or the query fails, the name will be listed as '<unknown>'.
"""
for cls_cmd, idn_cmd in [('*CLS', '*IDN?'), ('', 'ID?'), ('CS', 'OI')]:
resp = None
name = UNKNOWN
p = self.client.servers[server].packet()
p.address(channel).timeout(Value(1,'s')).write(cls_cmd).query(idn_cmd)
print("Sending '" + idn_cmd + "' to " + str(server) + " " + str(channel))
try:
resp = (yield p.send()).query
except Exception:
print("No response to '" + idn_cmd + "' from " + str(server) + " " + str(channel))
continue
name = parseIDNResponse(resp, idn_cmd)
if name != UNKNOWN:
print(str(server) + " " + str(channel) + " '" + idn_cmd + "' response: '" + resp + "'")
print(str(server) + " " + str(channel) + " device name: '" + name + "'")
break
returnValue((name, resp))
def identifyDevice(self, server, channel, idn):
"""Try to identify a new device with all ident functions.
Returns the first name returned by a successful identification.
"""
@inlineCallbacks
def _doIdentifyDevice():
for identifier in list(self.identFunctions.keys()):
name = yield self.tryIdentFunc(server, channel, idn, identifier)
if name is not None:
returnValue(name)
returnValue(UNKNOWN)
return self.identLock.run(_doIdentifyDevice)
def identifyDevicesWithServer(self, identifier):
"""Try to identify all unknown devices with a new server."""
@inlineCallbacks
def _doServerIdentify():
#yield self.client.refresh()
for (server, channel), (device, idn) in list(self.knownDevices.items()):
if device != UNKNOWN:
continue
name = yield self.tryIdentFunc(server, channel, idn, identifier)
if name is None:
continue
self.knownDevices[server, channel] = (name, idn)
if name in self.deviceServers:
self.notifyServers(name, server, channel, True)
return self.identLock.run(_doServerIdentify)
@inlineCallbacks
def tryIdentFunc(self, server, channel, idn, identifier):
"""Try calling one registered identification function.
If the identification succeeds, returns the new name,
otherwise returns None.
"""
if identifier in self.identFunctions:
s = self.client[identifier]
setting, context = self.identFunctions[identifier]
print("Trying to identify device " + str(server) + " " + str(channel) + " on server " + str(identifier))
try:
if idn is None:
resp = yield s[setting](server, channel, context=context)
else:
resp = yield s[setting](server, channel, idn, context=context)
except Exception:
print('Error while attempting to identify a device')
returnValue(UNKNOWN)
if resp is not None and resp != UNKNOWN:
print("Server " + str(identifier) + ' identified device ' + str(server) +
' ' + str(channel) + ' as ' + str(resp))
returnValue(resp)
else:
print("Server " + str(identifier) + ' could not identify device ' + str(server) + ' ' + str(channel))
@setting(1, 'Register Server',
devices=['s', '*s'], messageID='w',
returns='*(s{device} s{server} s{address}, b{isConnected})')
def register_server(self, c, devices, messageID):
"""Register as a server that handles a particular GPIB device(s).
Returns a list with information about all matching devices that
have been connected up to this point. After registering,
messages will be sent to the registered message ID whenever
a matching device connects or disconnects. The clusters sent
in response to this setting and those sent as messages have the same
format. For messages, the final boolean indicates whether the
device has been connected or disconnected, while in response to
this function call, the final boolean is always true, since we
only send info about connected devices.
The device name is determined by parsing the response to a *IDN?
query. To handle devices that don't support *IDN? correctly, use
the 'Register Ident Function' in addition.
"""
if isinstance(devices, str):
devices = [devices]
found = []
for device in devices:
servers = self.deviceServers.setdefault(device, [])
servers.append({'target': c.source,
'context': c.ID,
'messageID': messageID})
# gather info about matching servers already connected
for (server, channel), (known_device, idnResult) in self.knownDevices.items():
if device != known_device:
continue
found.append((device, server, channel, True))
return found
@setting(2, 'Register Ident Function', setting=['s', 'w'])
def register_ident_function(self, c, setting):
"""Specify a setting to be called to identify devices.
This setting must accept either of the following:
s, s, s: server, address, *IDN? response
s, s: server, address
If a device returned a non-standard response to a *IDN? query
(including possibly an empty string), then the first call signature
will be used. If the *IDN? query timed out or otherwise failed,
the second call signature will be used. As a server writer, you
must choose which of these signatures to support. Note that if the
device behavior is unpredictable (sometimes it returns a string,
sometimes it times out), you may need to support both signatures.
"""
self.identFunctions[c.source] = setting, c.ID
@setting(10)
def dump_info(self, c):
"""Returns information about the server status.
This info includes currently known devices, registered device
servers, and registered identification functions.
"""
return (str(self.knownDevices),
str(self.deviceServers),
str(self.identFunctions))
def notifyServers(self, device, server, channel, isConnected):
"""Notify all registered servers about a device status change."""
for s in self.deviceServers[device]:
rec = s['messageID'], (device, server, channel, isConnected)
print 'Sending message:', s['target'], s['context'], [rec]
self.client._sendMessage(s['target'], [rec], context=s['context'])
def serverConnected(self, ID, name):
"""New GPIBManagedServer's will register directly with us, before they
have even completed their registration with the LabRAD manager as a server.
We will get this signal once they are accessible through the LabRAD client
so that we can probe them for devices. This ordering matters mainly if
the new server has a custom IDN parsing function"""
# This was created to identify devices on a given server once
# it has completed its registration with the LabRAD manager. Before this
# device registration would fail if the server had a custom IDN handling
# function because this setting could not be properly accessed through the
# LabRAD manager at the time of execution.
recognizeServer = False
for device, serverInfo in list(self.deviceServers.items()):
if serverInfo[0]['target'] == ID:
recognizeServer = True
if recognizeServer:
callLater(0, self.identifyDevicesWithServer, ID)
def serverDisconnected(self, ID, name):
"""Disconnect devices when a bus server disconnects."""
for (server, channel) in list(self.knownDevices.keys()):
if server == name:
self.gpib_device_disconnect(server, channel)
def expireContext(self, c):
"""Stop sending notifications when a context expires."""
print 'Expiring context:', c.ID
# device servers
deletions = []
for device, servers in list(self.deviceServers.items()):
# remove all registrations with this context
servers = [s for s in servers if s['context'] != c.ID]
self.deviceServers[device] = servers
# if no one is left listening to this device, delete the list
if not len(servers):
deletions.append(device)
for device in deletions:
del self.deviceServers[device]
# ident functions
deletions = []
for src, idents in list(self.identFunctions.items()):
# remove all registrations with this context
idents = [i for i in idents if i[1] != c.ID]
self.identFunctions[src] = idents
# if no one is left listening to this device, delete the list
if not len(idents):
deletions.append(src)
for src in deletions:
del self.identFunctions[src]
class HendrixDeploy(object):
"""
HendrixDeploy encapsulates the necessary information needed to deploy
the HendrixService on a single or multiple processes.
"""
def __init__(self,
action='start',
options={},
reactor=reactor,
threadpool=None):
self.action = action
self.options = hx_options()
self.options.update(options)
self.services = []
self.resources = []
self.reactor = reactor
self.threadpool = threadpool or ThreadPool(name="Hendrix Web Service")
self.use_settings = True
# because running the management command overrides self.options['wsgi']
if self.options['wsgi']:
if hasattr(self.options['wsgi'], '__call__'):
# If it has a __call__, we assume that it is the application
# object itself.
self.application = self.options['wsgi']
try:
self.options['wsgi'] = "%s.%s" % (
self.application.__module__, self.application.__name__)
except AttributeError:
self.options['wsgi'] = self.application.__class__.__name__
else:
# Otherwise, we'll try to discern an application in the belief
# that this is a dot path.
wsgi_dot_path = self.options['wsgi']
# will raise AttributeError if we can't import it.
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.use_settings = False
else:
os.environ['DJANGO_SETTINGS_MODULE'] = self.options['settings']
settings = import_string('django.conf.settings')
self.services = get_additional_services(settings)
self.resources = get_additional_resources(settings)
self.options = HendrixDeploy.getConf(settings, self.options)
if self.use_settings:
django = importlib.import_module('django')
if django.VERSION[:2] >= (1, 7):
django.setup()
wsgi_dot_path = getattr(settings, 'WSGI_APPLICATION', None)
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.is_secure = self.options['key'] and self.options['cert']
self.servers = []
self._lock = DeferredLock()
@classmethod
def importWSGI(cls, wsgi_dot_path):
try:
wsgi_module, application_name = wsgi_dot_path.rsplit('.', 1)
except AttributeError:
pid = os.getpid()
chalk.red("Unable to discern a WSGI application from '%s'" %
wsgi_dot_path)
os.kill(pid, 15)
try:
wsgi = importlib.import_module(wsgi_module)
except ImportError:
chalk.red("Unable to Import module '%s'\n" % wsgi_dot_path)
raise
return getattr(wsgi, application_name, None)
@classmethod
def getConf(cls, settings, options):
"updates the options dict to use config options in the settings module"
ports = ['http_port', 'https_port', 'cache_port']
for port_name in ports:
port = getattr(settings, port_name.upper(), None)
# only use the settings ports if the defaults were left unchanged
default = getattr(defaults, port_name.upper())
if port and options.get(port_name) == default:
options[port_name] = port
_opts = [('key', 'hx_private_key'), ('cert', 'hx_certficate'),
('wsgi', 'wsgi_application')]
for opt_name, settings_name in _opts:
opt = getattr(settings, settings_name.upper(), None)
if opt:
options[opt_name] = opt
if not options['settings']:
options['settings'] = environ['DJANGO_SETTINGS_MODULE']
return options
def addServices(self):
"""
a helper function used in HendrixDeploy.run
it instanstiates the HendrixService and adds child services
note that these services will also be run on all processes
"""
self.addHendrix()
def addGlobalServices(self):
"""
This is where we put service that we don't want to be duplicated on
worker subprocesses
"""
pass
def getThreadPool(self):
'''
Case to match twisted.internet.reactor
'''
return self.threadpool
def addHendrix(self):
'''
Instantiates a HendrixService with this object's threadpool.
It will be added as a service later.
'''
self.hendrix = HendrixService(self.application,
threadpool=self.getThreadPool(),
resources=self.resources,
services=self.services,
loud=self.options['loud'])
if self.options["https_only"] is not True:
self.hendrix.spawn_new_server(self.options['http_port'],
HendrixTCPService)
def catalogServers(self, hendrix):
"collects a list of service names serving on TCP or SSL"
for service in hendrix.services:
if isinstance(service, (TCPServer, SSLServer)):
self.servers.append(service.name)
def _listening_message(self):
message = "non-TLS listening on port {}".format(
self.options['http_port'])
return message
def run(self):
"sets up the desired services and runs the requested action"
self.addServices()
self.catalogServers(self.hendrix)
action = self.action
fd = self.options['fd']
if action.startswith('start'):
chalk.blue(self._listening_message())
getattr(self, action)(fd)
###########################
# annnnd run the reactor! #
###########################
self.reactor.run()
elif action == 'restart':
getattr(self, action)(fd=fd)
else:
getattr(self, action)()
@property
def pid(self):
"The default location of the pid file for process management"
return get_pid(self.options)
def getSpawnArgs(self):
_args = [
'hx',
'start', # action
# kwargs
'--http_port',
str(self.options['http_port']),
'--https_port',
str(self.options['https_port']),
'--cache_port',
str(self.options['cache_port']),
'--workers',
'0',
'--fd',
pickle.dumps(self.fds),
]
# args/signals
if self.options['dev']:
_args.append('--dev')
if not self.use_settings:
_args += ['--wsgi', self.options['wsgi']]
return _args
def start(self, fd=None):
if fd is None:
# anything in this block is only run once
self.addGlobalServices()
self.hendrix.startService()
pids = [str(os.getpid())] # script pid
if self.options['workers']:
self.launchWorkers(pids)
self.pid_file = self.openPidList(pids)
else:
fds = pickle.loads(fd)
factories = {}
for name in self.servers:
factory = self.disownService(name)
factories[name] = factory
self.hendrix.startService()
for name, factory in factories.iteritems():
self.addSubprocess(fds, name, factory)
chalk.eraser()
chalk.blue('Starting Hendrix...')
def setFDs(self):
"""
Iterator for file descriptors.
Seperated from launchworkers for clarity and readability.
"""
# 0 corresponds to stdin, 1 to stdout, 2 to stderr
self.childFDs = {0: 0, 1: 1, 2: 2}
self.fds = {}
for name in self.servers:
self.port = self.hendrix.get_port(name)
fd = self.port.fileno()
self.childFDs[fd] = fd
self.fds[name] = fd
def launchWorkers(self, pids):
# Create a new listening port and several other processes to
# help out.
self.setFDs()
args = self.getSpawnArgs()
transports = []
for i in range(self.options['workers']):
time.sleep(0.05)
transport = self.reactor.spawnProcess(DeployServerProtocol(args),
'hx',
args,
childFDs=self.childFDs,
env=environ)
transports.append(transport)
pids.append(str(transport.pid))
def openPidList(self, pids):
with open(self.pid, 'w') as pid_file:
pid_file.write('\n'.join(pids))
return pid_file
def addSubprocess(self, fds, name, factory):
"""
Public method for _addSubprocess.
Wraps reactor.adoptStreamConnection in
a simple DeferredLock to guarantee
workers play well together.
"""
self._lock.run(self._addSubprocess, self, fds, name, factory)
def _addSubprocess(self, fds, name, factory):
self.reactor.adoptStreamConnection(fds[name], AF_INET, factory)
def stop(self, sig=9):
with open(self.pid) as pid_file:
pids = pid_file.readlines()
for pid in pids:
try:
os.kill(int(pid), sig)
except OSError:
# OSError raised when it trys to kill the child processes
pass
os.remove(self.pid)
chalk.green('Stopping Hendrix...')
def start_reload(self, fd=None):
self.start(fd=fd)
def restart(self, fd=None):
self.stop()
time.sleep(1) # wait a second to ensure the port is closed
self.start(fd)
def disownService(self, name):
"""
disowns a service on hendirix by name
returns a factory for use in the adoptStreamPort part of setting up
multiple processes
"""
_service = self.hendrix.getServiceNamed(name)
_service.disownServiceParent()
return _service.factory
def add_non_tls_websocket_service(self, websocket_factory):
autobahn.twisted.websocket.listenWS(websocket_factory)
class WampSnapshotComponent(ApplicationSession):
def __init__(self,
kernel_key,
config=ComponentConfig(realm=unicode("jupyter")),
path="data"):
ApplicationSession.__init__(self, config=config)
self._kernel_key = kernel_key
self.path = path
self._snapshots = {}
self._lut = {}
self._pending_tables = defaultdict(set)
self._lock = DeferredLock() # global lock
self._iterlocks = defaultdict(DeferredLock) # local locks
self._mount_locks = defaultdict(DeferredLock) # mount locks
self._sync_locks = defaultdict(DeferredLock) # sync locks
self._automount_queue = DeferredQueue()
self._automount()
def __del__(self):
ApplicationSession.__del__(self)
@inlineCallbacks
def onJoin(self, details):
log.msg("[WampSnapshotComponent] onJoin()")
log.msg(
"[WampSnapshotComponent] Registering Procedure: io.timbr.kernel.{}.snapd.list"
.format(self._kernel_key))
yield self.register(
self.list,
'io.timbr.kernel.{}.snapd.list'.format(self._kernel_key))
log.msg(
"[WampSnapshotComponent] Registering Procedure: io.timbr.kernel.{}.snapd.mount"
.format(self._kernel_key))
yield self.register(
self.mount,
'io.timbr.kernel.{}.snapd.mount'.format(self._kernel_key))
log.msg(
"[WampSnapshotComponent] Registering Procedure: io.timbr.kernel.{}.snapd.unmount"
.format(self._kernel_key))
yield self.register(
self.unmount,
'io.timbr.kernel.{}.snapd.unmount'.format(self._kernel_key))
log.msg(
"[WampSnapshotComponent] Registering Procedure: io.timbr.kernel.{}.snapd.fetch"
.format(self._kernel_key))
yield self.register(
self.fetch,
'io.timbr.kernel.{}.snapd.fetch'.format(self._kernel_key),
RegisterOptions(details_arg='details'))
log.msg(
"[WampSnapshotComponent] Registering Procedure: io.timbr.kernel.{}.snapd.get"
.format(self._kernel_key))
yield self.register(
self.get,
'io.timbr.kernel.{}.snapd.get'.format(self._kernel_key))
log.msg(
"[WampSnapshotComponent] Registering Procedure: io.timbr.kernel.{}.snapd.sync"
.format(self._kernel_key))
yield self.register(
self.sync,
'io.timbr.kernel.{}.snapd.sync'.format(self._kernel_key))
log.msg(
"[WampSnapshotComponent] Registering Procedure: io.timbr.kernel.{}.snapd.build_table"
.format(self._kernel_key))
yield self.register(
self.build_table,
'io.timbr.kernel.{}.snapd.build_table'.format(
self._kernel_key), RegisterOptions(details_arg='details'))
log.msg(
"[WampSnapshotComponent] Registering Procedure: io.timbr.kernel.{}.snapd.mark_as_dirty"
.format(self._kernel_key))
yield self.register(
self.mark_as_dirty,
'io.timbr.kernel.{}.snapd.mark_as_dirty'.format(
self._kernel_key))
@inlineCallbacks
def list(self):
# NOTE: to protect against changing the _snapshots dict while method is running
@inlineCallbacks
def critical():
res = {}
for k in self._snapshots:
res[k] = yield self.get(k)
returnValue(res)
result = yield self._lock.run(critical)
returnValue(result)
@inlineCallbacks
def mount(self, path, remount=False):
# NOTE: to protect against changing the _snapshots, _lut dicts while method is running
key = self._lut.get(path, str(ObjectId()))
def critical(key=key):
if remount and key in self._snapshots:
self._snapshots[key].close()
self._snapshots[key] = Snapshot(path)
self._pending_tables[key] = set(
self._snapshots[key].pending_tables)
elif key not in self._snapshots:
try:
self._snapshots[key] = Snapshot(path)
self._pending_tables[key] = set(
self._snapshots[key].pending_tables)
except (IOError, ValueError) as ne:
_logger.warn("Unable to mount snapshot %s: %s" %
(path, str(ne)))
return None
self._lut[path] = key
else:
# NOTE: duplicates pre-lock check functionality
warnings.warn("Path %s already mounted with key '%s'" %
(path, key))
return key
@inlineCallbacks
def safe_mount(key=key):
# NOTE: Acquiring this local lock ensures that there will be no concurrency issues with other
# methods performing operations on the snapshot. Why exactly this is needed would be good to know
result = yield self._iterlocks[key].run(critical)
returnValue(result)
if (remount is False and key in self._snapshots):
returnValue(key)
else:
result = yield self._lock.run(safe_mount)
returnValue(result)
@inlineCallbacks
def unmount(self, key):
# NOTE: to protect against changing the _snapshots, _lut dicts while method is running
def critical():
del self._lut[self._snapshots[key]._filename]
self._snapshots[key].close()
del self._snapshots[key]
try:
del self._pending_tables[key]
except KeyError as ke:
pass # no locks have been used for this key
try:
del self._sync_locks[key]
except KeyError as ke:
pass # no locks have been used for this key
if key in self._snapshots:
yield self._iterlocks[key].run(critical)
del self._iterlocks[key]
returnValue(True)
else:
result = yield False
returnValue(result)
@inlineCallbacks
def fetch(self, key, start=None, stop=None, step=1, details=None):
def critical(ktbl):
# NOTE: to protect against changing the _snapshots dicts while method is running
key, tbl = ktbl
if tbl is None:
ref = self._snapshots[key]
elif tbl not in self._snapshots[key].tables:
raise KeyError("Table '%s' not found in snapshot '%s'" %
(tbl, key))
else:
ref = self._snapshots[key].tbl[tbl]
return ref
def generate(kref):
# NOTE: to protect against reuse of the underlying pytables iterator by another call
key, ref = kref
if start is None and stop is None and step is 1:
for row in ref:
yield details.progress(make_wamp_safe(row))
else:
for row in ref.__iter__(slice(start, stop, step)):
yield details.progress(make_wamp_safe(row))
returnValue(None)
tbl = None
if isinstance(key, (list, tuple)):
assert len(
key
) == 2, "When argument 1 is a list or tuple, it must have 2 elements"
key, tbl = key
assert key in self._snapshots
ref = yield self._lock.run(critical, [key, tbl])
def iterate():
return cooperate(generate([key, ref])).whenDone()
yield self._iterlocks[key].run(iterate)
returnValue(None)
def get(self, key):
return {
"filename": self._snapshots[key]._filename,
"length": len(self._snapshots[key]),
"tables": self._snapshots[key].tables,
"pending_tables": list(self._pending_tables[key])
}
@inlineCallbacks
def sync(self, key):
_logger.debug("Sync called with key: %s" % key)
@inlineCallbacks
def critical():
# NOTE: protecting individual snapshot against modification because we will be opening
# it rw and modifying it during the sync process. Therefore iterating or getting info
# about it is a bad idea while this is running.
_logger.debug(" Sync iterlock['%s'] acquired." % key)
if key in self._snapshots:
# Execute using deferToThread after all locks have been acquired
try:
result = yield threads.deferToThread(
self._snapshots[key].sync)
self._pending_tables[key] = set(
self._snapshots[key].pending_tables)
returnValue(result)
except IncompleteSyncError as e:
pass
result = yield self._lock.run(critical)
returnValue(result)
@inlineCallbacks
def build_table(self, key, name, tblspec, details=None):
self._pending_tables[key].add(name)
s = self.get(key)
if s is None:
raise KeyError(
"Snapshot with key %s doesn't exist or is not mounted." %
key)
snapfile = s["filename"]
async_result = _process_pool.apply_async(_build_table_work_fn,
[snapfile, name, tblspec])
yield threads.deferToThread(async_result.get)
yield self._sync_locks[key].run(self.sync, key)
returnValue(True)
@inlineCallbacks
def mark_as_dirty(self, path):
if (not self._mount_locks[path].locked
and path not in self._automount_queue.pending):
yield self._automount_queue.put(path)
@inlineCallbacks
def _automount_one(self, path):
yield self._mount_locks[path].acquire()
_logger.debug("Automount processing %s" % path)
try:
if path in self._lut:
yield self.mount(path, remount=True)
else:
yield self.mount(path)
except tables.HDF5ExtError as e:
# empty or (temporarily?) corrupt file
_logger.debug("Empty or temporarily corrupt file %s: %s" %
(path, str(e)))
except ValueError as ve:
# the file is already open somewhere else
_logger.debug("File %s already open somewhere else: %s" %
(path, str(ve)))
self._mount_locks[path].release()
del self._mount_locks[path]
returnValue(path)
@inlineCallbacks
def _automount(self):
while True:
path = yield self._automount_queue.get()
self._automount_one(path)
def onLeave(self, details):
log.msg("[WampSnapshotComponent] onLeave()")
log.msg("details: %s" % str(details))
super(self.__class__, self).onLeave(details)
def onDisconnect(self):
log.msg("WampSnapshotComponent] onDisconnect()")
class WampPackagerComponent(ApplicationSession):
def __init__(self, kernel_key, conda_env=conda_env, env_path=env_path, pkg_path=pkg_path, config=ComponentConfig(realm=u"jupyter")):
ApplicationSession.__init__(self, config=config)
self._kernel_key = kernel_key
self._conda_env = conda_env
self._env_path = env_path
self._pkg_path = pkg_path
self._conda_env = conda_env
self._lock = DeferredLock()
@inlineCallbacks
def install_pkg(self, pkg_type, pkg, details):
if str(pkg_type) == 'pip':
pipcmd = sh.Command(os.path.join([self._env_path, '{}/bin/pip'.format(self._conda_env)]))
yield cooperative_consume( pipcmd("install", pkg, _iter_noblock=True), details.progress)
else:
yield cooperative_consume( sh.conda("install", "--name", self._conda_env, pkg, "-y", _iter_noblock=True), details.progress)
returnValue(None)
@inlineCallbacks
def onJoin(self, details):
log.msg("[WampPackagerComponent] onJoin()")
@inlineCallbacks
def install_environment(pkg_name, details=None):
@inlineCallbacks
def install():
pkg_filepath = os.path.join(self._pkg_path, "%s.yaml" % pkg_name)
if not os.path.exists(pkg_filepath):
raise IOError("Package specification file :: %s :: not found" % pkg_filepath)
else:
with open(pkg_filepath) as env:
pkgs = yaml.load(env)
conda_deps = ["=".join(p.split("=")[:1]) for p in pkgs['dependencies'] if isinstance(p, StringType)]
if conda_deps:
yield self.install_pkg('conda', conda_deps, details)
pip_deps = [x for x in pkgs['dependencies'] if isinstance(x, dict) and 'pip' in x]
if pip_deps:
yield self.install_pkg('pip', pip_deps[0]['pip'], details)
returnValue(None)
yield self._lock.run(install)
returnValue(None)
log.msg("[WampPackagerComponent] Registering Procedure: io.timbr.kernel.{}.pkgd.install_environment".format(self._kernel_key))
yield self.register(install_environment, 'io.timbr.kernel.{}.pkgd.install_environment'.format(self._kernel_key), RegisterOptions(details_arg = 'details'))
@inlineCallbacks
def install_package(pkg_type, pkg_name, details=None):
@inlineCallbacks
def install():
yield self.install_pkg(pkg_type, pkg_name, details)
yield self._lock.run(install)
returnValue(None)
log.msg("[WampPackagerComponent] Registering Procedure: io.timbr.kernel.{}.pkgd.install_package".format(self._kernel_key))
yield self.register(install_package, 'io.timbr.kernel.{}.pkgd.install_package'.format(self._kernel_key), RegisterOptions(details_arg = 'details'))
@inlineCallbacks
def package_environment(pkg_name, details=None):
@inlineCallbacks
def package():
pkg_filepath = os.path.join(self._pkg_path, "%s.yaml" %(pkg_name))
yield cooperative_consume(
sh.conda("env", "export", "-n", self._conda_env, "--file", pkg_filepath, _iter_noblock=True), details.progress)
returnValue(None)
yield self._lock.run(package)
returnValue(None)
log.msg("[WampPackagerComponent] Registering Procedure: io.timbr.kernel.{}.pkgd.package_environment".format(self._kernel_key))
yield self.register(package_environment, 'io.timbr.kernel.{}.pkgd.package_environment'.format(self._kernel_key), RegisterOptions(details_arg = 'details'))
def onLeave(self, details):
log.msg("[WampPackagerComponent] onLeave()" )
log.msg("details: %s" % str(details))
reactor.callLater(0.25, _packager_runner.run, build_packager_component(self._kernel_key, self._conda_env), start_reactor=False)
def onDisconnect(self):
log.msg("[WampPackagerComponent] onDisconnect()")
class GPIBDeviceManager(LabradServer):
"""Manages autodetection and identification of GPIB devices.
The device manager listens for "GPIB Device Connect" and
"GPIB Device Disconnect" messages coming from GPIB bus servers.
It attempts to identify the connected devices and forward the
messages on to servers interested in particular devices. For
devices that cannot be identified by *IDN? in the usual way,
servers can register an identification setting to be called
by the device manager to properly identify the device.
"""
name = 'GPIB Device Manager'
@inlineCallbacks
def initServer(self):
"""Initialize the server after connecting to LabRAD."""
self.knownDevices = {} # maps (server, channel) to (name, idn)
self.deviceServers = {} # maps device name to list of interested servers.
# each interested server is {'target':<>,'context':<>,'messageID':<>}
self.identFunctions = {} # maps server to (setting, ctx) for ident
self.identLock = DeferredLock()
# named messages are sent with source ID first, which we ignore
#connect_func = lambda c, (s, payload): self.gpib_device_connect(*payload)
#disconnect_func = lambda c, (s, payload): self.gpib_device_disconnect(*payload)
connect_func = lambda c, d: self.gpib_device_connect(*d[1])
disconnect_func = lambda c, d: self.gpib_device_disconnect(*d[1])
mgr = self.client.manager
self._cxn.addListener(connect_func, source=mgr.ID, ID=10)
self._cxn.addListener(disconnect_func, source=mgr.ID, ID=11)
yield mgr.subscribe_to_named_message('GPIB Device Connect', 10, True)
yield mgr.subscribe_to_named_message('GPIB Device Disconnect', 11, True)
# do an initial scan of the available GPIB devices
yield self.refreshDeviceLists()
@inlineCallbacks
def refreshDeviceLists(self):
"""Ask all GPIB bus servers for their available GPIB devices."""
servers = [s for n, s in self.client.servers.items()
if (('GPIB Bus' in n) or ('gpib_bus' in n)) and \
(('List Devices' in s.settings) or \
('list_devices' in s.settings))]
serverNames = [s.name for s in servers]
print('Pinging servers:', serverNames)
resp = yield DeferredList([s.list_devices() for s in servers])
for serverName, (success, addrs) in zip(serverNames, resp):
if not success:
print('Failed to get device list for:', serverName)
else:
print('Server %s has devices: %s' % (serverName, addrs))
for addr in addrs:
self.gpib_device_connect(serverName, addr)
@inlineCallbacks
def gpib_device_connect(self, gpibBusServer, channel):
"""Handle messages when devices connect."""
print('Device Connect:', gpibBusServer, channel)
if (gpibBusServer, channel) in self.knownDevices:
return
device, idnResult = yield self.lookupDeviceName(gpibBusServer, channel)
if device == UNKNOWN:
device = yield self.identifyDevice(gpibBusServer, channel, idnResult)
self.knownDevices[gpibBusServer, channel] = (device, idnResult)
# forward message if someone cares about this device
if device in self.deviceServers:
self.notifyServers(device, gpibBusServer, channel, True)
def gpib_device_disconnect(self, server, channel):
"""Handle messages when devices connect."""
print('Device Disconnect:', server, channel)
if (server, channel) not in self.knownDevices:
return
device, idnResult = self.knownDevices[server, channel]
del self.knownDevices[server, channel]
# forward message if someone cares about this device
if device in self.deviceServers:
self.notifyServers(device, server, channel, False)
@inlineCallbacks
def lookupDeviceName(self, server, channel):
"""Try to send a *IDN? query to lookup info about a device.
Returns the name of the device and the actual response string
to the identification query. If the response cannot be parsed
or the query fails, the name will be listed as '<unknown>'.
"""
for k in range(0,2):
p = self.client.servers[server].packet()
p.address(channel).timeout(Value(1,'s')).write('*CLS').write('*IDN?').read()
print('Sending *IDN? to', server, channel)
resp = None
try:
resp = (yield p.send()).read
name = parseIDNResponse(resp)
except Exception as e:
print('Error sending *IDN? to', server, channel + ':', e)
name = UNKNOWN
returnValue((name, resp))
def identifyDevice(self, server, channel, idn):
"""Try to identify a new device with all ident functions.
Returns the first name returned by a successful identification.
"""
@inlineCallbacks
def _doIdentifyDevice():
for identifier in list(self.identFunctions.keys()):
name = yield self.tryIdentFunc(server, channel, idn, identifier)
if name is not None:
returnValue(name)
returnValue(UNKNOWN)
return self.identLock.run(_doIdentifyDevice)
def identifyDevicesWithServer(self, identifier):
"""Try to identify all unknown devices with a new server."""
@inlineCallbacks
def _doServerIdentify():
#yield self.client.refresh()
for (server, channel), (device, idn) in list(self.knownDevices.items()):
if device != UNKNOWN:
continue
name = yield self.tryIdentFunc(server, channel, idn, identifier)
if name is None:
continue
self.knownDevices[server, channel] = (name, idn)
if name in self.deviceServers:
self.notifyServers(name, server, channel, True)
return self.identLock.run(_doServerIdentify)
@inlineCallbacks
def tryIdentFunc(self, server, channel, idn, identifier):
"""Try calling one registered identification function.
If the identification succeeds, returns the new name,
otherwise returns None.
"""
try:
#yield self.client.refresh()
s = self.client[identifier]
setting, context = self.identFunctions[identifier]
print('Trying to identify device', server, channel)
print('on server', identifier)
print('with *IDN?:', repr(idn))
if idn is None:
resp = yield s[setting](server, channel, context=context)
else:
resp = yield s[setting](server, channel, idn, context=context)
if resp is not None:
data = (identifier, server, channel, resp)
print('Server %s identified device %s %s as "%s"' % data)
returnValue(resp)
except Exception as e:
print('Error during ident:', str(e))
@setting(1, 'Register Server',
devices=['s', '*s'], messageID='w',
returns='*(s{device} s{server} s{address}, b{isConnected})')
def register_server(self, c, devices, messageID):
"""Register as a server that handles a particular GPIB device(s).
Returns a list with information about all matching devices that
have been connected up to this point:
[(device name, gpib server name, gpib channel, bool)]
After registering, messages will be sent to the registered
message ID whenever a matching device connects or disconnects.
The clusters sent in response to this setting and those sent as
messages have the same format. For messages, the final boolean
indicates whether the device has been connected or disconnected,
while in response to this function call, the final boolean is
always true, since we only send info about connected devices.
The device name is determined by parsing the response to a *IDN?
query. To handle devices that don't support *IDN? correctly, use
the 'Register Ident Function' in addition.
"""
#Managed device servers can specify device names as string or a list of strings
if isinstance(devices, str):
devices = [devices]
found = []
for device in devices:
servers = self.deviceServers.setdefault(device, [])
servers.append({'target': c.source,
'context': c.ID,
'messageID': messageID})
# gather info about matching servers already connected
for (server, channel), (known_device, idnResult) in self.knownDevices.items():
if device != known_device:
continue
found.append((device, server, channel, True))
return found
@setting(2, 'Register Ident Function', setting=['s', 'w'])
def register_ident_function(self, c, setting):
"""Specify a setting to be called to identify devices.
This setting must accept either of the following:
s, s, s: server, address, *IDN? response
s, s: server, address
If a device returned a non-standard response to a *IDN? query
(including possibly an empty string), then the first call signature
will be used. If the *IDN? query timed out or otherwise failed,
the second call signature will be used. As a server writer, you
must choose which of these signatures to support. Note that if the
device behavior is unpredictable (sometimes it returns a string,
sometimes it times out), you may need to support both signatures.
"""
self.identFunctions[c.source] = setting, c.ID
@setting(10)
def dump_info(self, c):
"""Returns information about the server status.
This info includes currently known devices, registered device
servers, and registered identification functions.
"""
return (str(self.knownDevices),
str(self.deviceServers),
str(self.identFunctions))
def notifyServers(self, device, server, channel, isConnected):
"""Notify all registered servers about a device status change."""
for s in self.deviceServers[device]:
rec = s['messageID'], (device, server, channel, isConnected)
print('Sending message:', s['target'], s['context'], [rec])
self.client._sendMessage(s['target'], [rec], context=s['context'])
def serverConnected(self, ID, name):
"""New GPIBManagedServers will register directly with us, before they
have even completed their registration with the LabRAD manager as a server.
We will get this signal once they are accessible through the LabRAD client
so that we can probe them for devices. This ordering matters mainly if
the new server has a custom IDN parsing function"""
recognizeServer = False
for device, serverInfo in list(self.deviceServers.items()):
if serverInfo[0]['target'] == ID:
recognizeServer = True
if recognizeServer:
callLater(0, self.identifyDevicesWithServer, ID)
def serverDisconnected(self, ID, name):
"""Disconnect devices when a bus server disconnects."""
for (server, channel) in list(self.knownDevices.keys()):
if server == name:
self.gpib_device_disconnect(server, channel)
def expireContext(self, c):
"""Stop sending notifications when a context expires."""
print('Expiring context:', c.ID)
# device servers
deletions = []
for device, servers in list(self.deviceServers.items()):
# remove all registrations with this context
servers = [s for s in servers if s['context'] != c.ID]
self.deviceServers[device] = servers
# if no one is left listening to this device, delete the list
if not len(servers):
deletions.append(device)
for device in deletions:
del self.deviceServers[device]
# ident functions
deletions = []
for src, idents in list(self.identFunctions.items()):
# remove all registrations with this context
idents = [i for i in idents if i[1] != c.ID]
self.identFunctions[src] = idents
# if no one is left listening to this device, delete the list
if not len(idents):
deletions.append(src)
for src in deletions:
del self.identFunctions[src]
class WorkQueue(object):
"""A WorkQueue contains WorkUnits and dispatches NonceRanges when requested
by the miner. WorkQueues dispatch deffereds when they runs out of nonces.
"""
def __init__(self, core):
self.core = core
self.logger = core.logger
self.queueSize = core.config.get("general", "queuesize", int, 1)
self.queueDelay = core.config.get("general", "queuedelay", int, 5)
self.lock = DeferredLock()
self.queue = deque("", self.queueSize)
self.deferredQueue = deque()
self.currentUnit = None
self.lastBlock = None
self.block = ""
self.staleCallbacks = []
def storeWork(self, aw):
# check if this work matches the previous block
if (self.lastBlock is not None) and (aw.identifier == self.lastBlock):
self.logger.debug("Server gave work from the previous " "block, ignoring.")
# if the queue is too short request more work
if self.checkQueue():
if self.core.connection:
self.core.connection.requestWork()
return
# create a WorkUnit
work = WorkUnit(aw)
reactor.callLater(max(60, aw.time - 1) - self.queueDelay, self.checkWork)
reactor.callLater(max(60, aw.time - 1), self.workExpire, work)
# check if there is a new block, if so reset queue
newBlock = aw.identifier != self.block
if newBlock:
self.queue.clear()
self.currentUnit = None
self.lastBlock = self.block
self.block = aw.identifier
self.logger.debug("New block (WorkQueue)")
# add new WorkUnit to queue
if work.data and work.target and work.midstate and work.nonces:
self.queue.append(work)
# if the queue is too short request more work
workRequested = False
if self.checkQueue():
if self.core.connection:
self.core.connection.requestWork()
workRequested = True
# if there is a new block notify kernels that their work is now stale
if newBlock:
for callback in self.staleCallbacks:
callback()
self.staleCallbacks = []
self.staleCallbacks.append(work.stale)
# check if there are deferred WorkUnit requests pending
# since requests to fetch a WorkUnit can add additional deferreds to
# the queue, cache the size beforehand to avoid infinite loops.
for i in range(len(self.deferredQueue)):
df = self.deferredQueue.popleft()
d = self.fetchUnit(workRequested)
d.chainDeferred(df)
# clear the idle flag since we just added work to queue
self.core.reportIdle(False)
def checkWork(self):
# Called 5 seconds before any work expires in order to fetch more
if self.checkQueue():
if self.core.connection:
self.core.requestWork()
def checkQueue(self, added=False):
# This function checks the queue length including the current unit
size = 1
# Check if the current unit will last long enough
if self.currentUnit is None:
if len(self.queue) == 0:
return True
else:
size = 0
if added:
rolls = self.queue[0].maxtime - self.queue[0].timestamp
# If new work can't be rolled, and queue would be too small
if rolls == 0 and (len(self.queue) - 1) < self.queueSize:
return True
else:
remaining = self.currentUnit.maxtime - self.currentUnit.timestamp
# Check if we are about to run out of rolltime on current unit
if remaining < (self.queueDelay):
size = 0
# Check if the current unit is about to expire
age = self.currentUnit.downloaded + self.currentUnit.time
lifetime = age - time()
if lifetime < (2 * self.queueDelay):
size = 0
# Check if the queue will last long enough
queueLength = 0
for i in range(len(self.queue)):
age = self.queue[0].downloaded + max(60, self.queue[0].time - 1)
lifetime = age - time()
if lifetime > (2 * self.queueDelay):
queueLength += 1
# Return True/False indicating if more work should be fetched
return size + queueLength < self.queueSize
def workExpire(self, wu):
# Don't expire WorkUnits if idle and queue empty
if (self.core.idle) and (len(self.queue) <= 1):
return
# Remove the WorkUnit from queue
if len(self.queue) > 0:
iSize = len(self.queue)
if not (len(self.queue) == 1 and (self.currentUnit is None)):
try:
self.queue.remove(wu)
except ValueError:
pass
if self.currentUnit == wu:
self.currentUnit = None
# Check queue size
if self.checkQueue() and (iSize != len(self.queue)):
if self.core.connection:
self.core.connection.requestWork()
# Flag the WorkUnit as stale
wu.stale()
else:
# Check back again later if we didn't expire the work
reactor.callLater(5, self.workExpire, wu)
def getRangeFromUnit(self, size):
# get remaining nonces
noncesLeft = self.currentUnit.nonces - self.currentUnit.base
# Flag indicating if the WorkUnit was depeleted by this request
depleted = False
# if there are enough nonces to fill the full reqest
if noncesLeft >= size:
nr = NonceRange(self.currentUnit, self.currentUnit.base, size)
# check if this uses up the rest of the WorkUnit
if size >= noncesLeft:
depleted = True
else:
self.currentUnit.base += size
# otherwise send whatever is left
else:
nr = NonceRange(self.currentUnit, self.currentUnit.base, noncesLeft)
depleted = True
# return the range
return nr, depleted
def checkRollTime(self, wu):
# This function checks if a WorkUnit could be time rolled
if wu.maxtime > wu.timestamp and not wu.isStale:
remaining = (wu.downloaded + wu.time) - time()
if remaining > (self.queueDelay) or len(self.queue) < 1:
# If it has been more than 5 minutes probably better to idle
if time() - wu.downloaded < 300:
return True
return False
def rollTime(self, wu):
# Check if this WorkUnit supports rolling time, return None if not
if not self.checkRollTime(wu):
return None
# Create the new WU
newWU = WorkUnit(wu)
# Increment the timestamp
newWU.timestamp += 1
# Reset the download time to the original WU's
newWU.downloaded = wu.downloaded
# Set a stale callback for this WU
self.staleCallbacks.append(newWU.stale)
# Setup a workExpire callback
remaining = max(self.queueDelay, (wu.downloaded + wu.time) - time())
reactor.callLater(remaining - 1, self.workExpire, newWU)
# Return the new WU
return newWU
def fetchUnit(self, delayed=False):
# if there is a unit in queue
if len(self.queue) >= 1:
# check if the queue has fallen below the desired size
if self.checkQueue(True) and (not delayed):
# Request more work to maintain minimum queue size
if self.core.connection:
self.core.connection.requestWork()
# get the next unit from queue
wu = self.queue.popleft()
# return the unit
return defer.succeed(wu)
# if the queue is empty
else:
# request more work
if self.core.connection:
self.core.connection.requestWork()
# report that the miner is idle
self.core.reportIdle(True)
# set up and return deferred
df = defer.Deferred()
self.deferredQueue.append(df)
return df
# make sure that only one fetchRange request runs at a time
def fetchRange(self, size=0x10000):
return self.lock.run(self._fetchRange, size)
def _fetchRange(self, size):
# make sure size is not too large
size = min(size, 0x100000000)
# check if the current unit exists
if self.currentUnit is not None:
# Get a nonce range
nr, depleated = self.getRangeFromUnit(size)
# If we depleted the Workunit then try to roll time
if depleated:
self.currentUnit = self.rollTime(self.currentUnit)
# Return the range
return defer.succeed(nr)
# if there is no current unit
else:
# Check if we can get a new unit with rolltime
def callback(wu):
# get a new current unit
self.currentUnit = wu
# get a nonce range
nr, depleated = self.getRangeFromUnit(size)
# If we depleted the Workunit then try to roll time
if depleated:
self.currentUnit = self.rollTime(self.currentUnit)
# return the range
return nr
d = self.fetchUnit()
d.addCallback(callback)
return d
class BNCSerial(BNCPulser):
@inlineCallbacks
def connect(self, server, port, baud):
self.server = server
#usage ensures only one read/write operation at a time
self._portLock = DeferredLock()
print 'connecting to "%s" on port "%s"...' % (server.name, port)
p = self.server.packet()
p.open(port)
p.baudrate(baud)
p.parity('N') #no parity bit
p.bytesize(8L)
p.stopbits(1L)
p.write_line(':SYST:COMM:SER:ECHO ON')
yield p.send()
chList = yield self.channel_list()
self.chMap = dict(chList)
print 'connected to %s...' % server.name
@staticmethod
def _queryGood(text, tag):
'''Check device response is complete.'''
#if last characters aren't newline, can't be done
if text[-2:] == '\r\n':
lines = text.split('\r\n')
#now ensure that echo and response lines present
#lines[-3:] should be [echo, response, '']
if len(lines) > 2:
if lines[-3] == tag:
return True
return False
@inlineCallbacks
def query(self, line):
yield self._portLock.acquire()
p = self.server.packet()
ret = yield p.write_line(line)\
.pause(U.Value(20.0, 'ms'))\
.read()\
.send()
text = ret['read']
while not self._queryGood(text, line):
add = yield self.server.read()
text = text + add
self._portLock.release()
returnValue(text.split('\r\n')[-2])
@inlineCallbacks
def send(self, line):
yield self._portLock.run(self.server.write_line, line)
@inlineCallbacks
def all_ch_states(self):
chns = self.chMap.keys()
p = self.server.packet()
p.read() #clear buffer
for n in chns:
p.write(':PULSE%d:STATE?' % n)
p.pause(U.Value(30, 'ms'))
p.read(key=str(n))
resp = yield p.send()
parser = TYPE_MAP['bool'][1]
returnValue([(n, parser(resp[str(n)])) for n in chns])
class CFProcessor(service.Service):
def __init__(self, name, conf):
_log.info("CF_INIT %s", name)
self.name, self.conf = name, conf
self.channel_dict = defaultdict(list)
self.iocs = dict()
self.client = None
self.currentTime = getCurrentTime
self.lock = DeferredLock()
def startService(self):
service.Service.startService(self)
# Returning a Deferred is not supported by startService(),
# so instead attempt to acquire the lock synchonously!
d = self.lock.acquire()
if not d.called:
d.cancel()
service.Service.stopService(self)
raise RuntimeError(
'Failed to acquired CF Processor lock for service start')
try:
self._startServiceWithLock()
except:
service.Service.stopService(self)
raise
finally:
self.lock.release()
def _startServiceWithLock(self):
_log.info("CF_START")
if self.client is None: # For setting up mock test client
"""
Using the default python cf-client. The url, username, and
password are provided by the channelfinder._conf module.
"""
from channelfinder import ChannelFinderClient
self.client = ChannelFinderClient()
try:
cf_props = [
prop['name'] for prop in self.client.getAllProperties()
]
if (self.conf.get('alias', 'default') == 'on'):
reqd_props = {
'hostName', 'iocName', 'pvStatus', 'time', 'iocid',
'alias'
}
else:
reqd_props = {
'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
}
wl = self.conf.get('infotags', list())
whitelist = [s.strip(', ') for s in wl.split()] \
if wl else wl
# Are any required properties not already present on CF?
properties = reqd_props - set(cf_props)
# Are any whitelisted properties not already present on CF?
# If so, add them too.
properties.update(set(whitelist) - set(cf_props))
owner = self.conf.get('username', 'cfstore')
for prop in properties:
self.client.set(property={u'name': prop, u'owner': owner})
self.whitelist = set(whitelist)
_log.debug('WHITELIST = {}'.format(self.whitelist))
except ConnectionError:
_log.exception("Cannot connect to Channelfinder service")
raise
else:
if self.conf.getboolean('cleanOnStart', True):
self.clean_service()
def stopService(self):
service.Service.stopService(self)
return self.lock.run(self._stopServiceWithLock)
def _stopServiceWithLock(self):
# Set channels to inactive and close connection to client
if self.conf.getboolean('cleanOnStop', True):
self.clean_service()
_log.info("CF_STOP")
# @defer.inlineCallbacks # Twisted v16 does not support cancellation!
def commit(self, transaction_record):
return self.lock.run(self._commitWithLock, transaction_record)
def _commitWithLock(self, TR):
self.cancelled = False
t = deferToThread(self._commitWithThread, TR)
def cancelCommit(d):
self.cancelled = True
d.callback(None)
d = defer.Deferred(cancelCommit)
def waitForThread(_ignored):
if self.cancelled:
return t
d.addCallback(waitForThread)
def chainError(err):
if not err.check(defer.CancelledError):
_log.error("CF_COMMIT FAILURE: %s", err)
if self.cancelled:
if not err.check(defer.CancelledError):
raise defer.CancelledError()
return err
else:
d.callback(None)
def chainResult(_ignored):
if self.cancelled:
raise defer.CancelledError()
else:
d.callback(None)
t.addCallbacks(chainResult, chainError)
return d
def _commitWithThread(self, TR):
if not self.running:
raise defer.CancelledError(
'CF Processor is not running (TR: %s:%s)', TR.src.host,
TR.src.port)
_log.info("CF_COMMIT: %s", TR)
"""
a dictionary with a list of records with their associated property info
pvInfo
{rid: { "pvName":"recordName",
"infoProperties":{propName:value, ...}}}
"""
host = TR.src.host
port = TR.src.port
iocName = TR.infos.get('IOCNAME') or TR.src.port
hostName = TR.infos.get('HOSTNAME') or TR.src.host
owner = TR.infos.get('ENGINEER') or TR.infos.get(
'CF_USERNAME') or self.conf.get('username', 'cfstore')
time = self.currentTime()
"""The unique identifier for a particular IOC"""
iocid = host + ":" + str(port)
pvInfo = {}
for rid, (rname, rtype) in TR.addrec.items():
pvInfo[rid] = {"pvName": rname}
for rid, (recinfos) in TR.recinfos.items():
# find intersection of these sets
if rid not in pvInfo:
_log.warn('IOC: %s: PV not found for recinfo with RID: %s',
iocid, rid)
continue
recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
if recinfo_wl:
pvInfo[rid]['infoProperties'] = list()
for infotag in recinfo_wl:
property = {
u'name': infotag,
u'owner': owner,
u'value': recinfos[infotag]
}
pvInfo[rid]['infoProperties'].append(property)
for rid, alias in TR.aliases.items():
if rid not in pvInfo:
_log.warn('IOC: %s: PV not found for alias with RID: %s',
iocid, rid)
continue
pvInfo[rid]['aliases'] = alias
delrec = list(TR.delrec)
_log.debug("Delete records: %s", delrec)
pvInfoByName = {}
for rid, (info) in pvInfo.items():
if info["pvName"] in pvInfoByName:
_log.warn(
"Commit contains multiple records with PV name: %s (%s)",
pv, iocid)
continue
pvInfoByName[info["pvName"]] = info
_log.debug("Add record: %s: %s", rid, info)
if TR.initial:
"""Add IOC to source list """
self.iocs[iocid] = {
"iocname": iocName,
"hostname": hostName,
"owner": owner,
"time": time,
"channelcount": 0
}
if not TR.connected:
delrec.extend(self.channel_dict.keys())
for pv in pvInfoByName.keys():
self.channel_dict[pv].append(
iocid) # add iocname to pvName in dict
self.iocs[iocid]["channelcount"] += 1
"""In case, alias exists"""
if (self.conf.get('alias', 'default' == 'on')):
if pv in pvInfoByName and "aliases" in pvInfoByName[pv]:
for a in pvInfoByName[pv]["aliases"]:
self.channel_dict[a].append(
iocid) # add iocname to pvName in dict
self.iocs[iocid]["channelcount"] += 1
for pv in delrec:
if iocid in self.channel_dict[pv]:
self.channel_dict[pv].remove(iocid)
if iocid in self.iocs:
self.iocs[iocid]["channelcount"] -= 1
if self.iocs[iocid]['channelcount'] == 0:
self.iocs.pop(iocid, None)
elif self.iocs[iocid]['channelcount'] < 0:
_log.error("Channel count negative: %s", iocid)
if len(self.channel_dict[pv]
) <= 0: # case: channel has no more iocs
del self.channel_dict[pv]
"""In case, alias exists"""
if (self.conf.get('alias', 'default' == 'on')):
if pv in pvInfoByName and "aliases" in pvInfoByName[pv]:
for a in pvInfoByName[pv]["aliases"]:
self.channel_dict[a].remove(iocid)
if iocid in self.iocs:
self.iocs[iocid]["channelcount"] -= 1
if self.iocs[iocid]['channelcount'] == 0:
self.iocs.pop(iocid, None)
elif self.iocs[iocid]['channelcount'] < 0:
_log.error("Channel count negative: %s", iocid)
if len(self.channel_dict[a]
) <= 0: # case: channel has no more iocs
del self.channel_dict[a]
poll(__updateCF__, self, pvInfoByName, delrec, hostName, iocName,
iocid, owner, time)
dict_to_file(self.channel_dict, self.iocs, self.conf)
def clean_service(self):
"""
Marks all channels as "Inactive" until the recsync server is back up
"""
sleep = 1
retry_limit = 5
owner = self.conf.get('username', 'cfstore')
while 1:
try:
_log.info("CF Clean Started")
channels = self.client.findByArgs(
prepareFindArgs(self.conf, [('pvStatus', 'Active')]))
if channels is not None:
new_channels = []
for ch in channels or []:
new_channels.append(ch[u'name'])
_log.info("Total channels to update: %s",
len(new_channels))
while len(new_channels) > 0:
_log.debug(
'Update "pvStatus" property to "Inactive" for %s channels',
min(len(new_channels), 10000))
self.client.update(property={
u'name': 'pvStatus',
u'owner': owner,
u'value': "Inactive"
},
channelNames=new_channels[:10000])
new_channels = new_channels[10000:]
_log.info("CF Clean Completed")
return
except RequestException as e:
_log.error("Clean service failed: %s", e)
_log.info("Clean service retry in %s seconds", min(60, sleep))
time.sleep(min(60, sleep))
sleep *= 1.5
if self.running == 0 and sleep >= retry_limit:
_log.info("Abandoning clean after %s seconds", retry_limit)
return
class Subscriber(object):
def __init__(self, connection, metadata):
self.connection = connection
self.metadata = metadata
self._sq = SubscriberQueue()
self._dependents = []
self._parents = []
self._dependency_info = {}
self._notify_lock = DeferredLock()
def get_id(self):
return self.metadata.get('client_tag')
def get_filter_exp(self):
return self.metadata.get('filter_exp')
def send_message(self, message_hash):
print (self.get_queue_name(), "Appending msg to my queueu ", self.get_id(), message_hash, self._sq.size())
self._sq.append(message_hash)
def get_message(self):
return self._sq.pop()
def is_independent(self):
return len(self._parents) == 0
def get_queue_name(self):
return self.metadata.get('queue_name')
def get_parents(self):
return self._parents
def get_dependents(self):
return self._dependents
def add_dependent(self, dependent):
self._dependents.append(dependent)
def add_parent(self, parent):
print("Add parent ", parent.get_id(), "for subscriber ", self.get_id())
self._parents.append(parent)
def notify_dependents(self, message_hash):
dependents = self.get_dependents()
if not dependents:
return
for dependent in dependents:
print("Notifying message ", message_hash, "to dependent ",
dependent.get_id())
dependent.notify(message_hash)
def notify(self, message_hash):
self._notify_lock.run(self._notify, message_hash)
def _notify(self, message_hash):
"""Notify this subscriber that it should process the given message.
This method is called when a parent subscriber receives an ACK for a
message it was processing.
"""
if message_hash not in self._dependency_info:
self._dependency_info[message_hash] = len(self._parents)
self._dependency_info[message_hash] -= 1
parents_left = self._dependency_info[message_hash]
if parents_left <= 0:
print("All parents have consumed the message ", message_hash,
"I will consume this now ", self.get_id())
self.send_message(message_hash)
