def __init__(self, stream):
"""
:param stream: The stream name.
:type stream: str
"""
self.stream = stream
self.queue = Queue(maxsize=100)
self.is_open = False
self.sequential = Sequential()
self.journal = {}
self.thread = Thread(target=self._open)
self.thread.setDaemon(True)
self.thread.start()
def run(self):
while not Thread.aborted():
sleep(self.INTERVAL)
try:
reply = protocol.Ping(self.pid)
reply.send(self.pipe)
except PipeBroken:
sys.exit(1)
def run(self):
while not Thread.aborted():
sleep(self.INTERVAL)
try:
reply = protocol.Ping(self.pid)
reply.send(self.pipe)
except PipeBroken:
sys.exit(1)
def _drain(self):
"""
Drain the queue.
"""
self.is_open = False
while not Thread.aborted():
try:
request = self.queue.get(timeout=1)
self.commit(request.sn)
except Empty:
break
def _drain(self):
"""
Drain the queue.
"""
self.is_open = False
while not Thread.aborted():
try:
request = self.queue.get(timeout=1)
self.commit(request.sn)
except Empty:
break
def get(self):
"""
Get the next pending request to be dispatched.
Blocks until a request is available.
:return: The next pending request.
:rtype: Document
"""
while not Thread.aborted():
try:
return self.queue.get(timeout=10)
except Empty:
pass
def get(self):
"""
Get the next pending request to be dispatched.
Blocks until a request is available.
:return: The next pending request.
:rtype: Document
"""
while not Thread.aborted():
try:
return self.queue.get(timeout=10)
except Empty:
pass
def __init__(self, stream):
"""
:param stream: The stream name.
:type stream: str
"""
self.stream = stream
self.queue = Queue(maxsize=100)
self.is_open = False
self.sequential = Sequential()
self.journal = {}
self.thread = Thread(target=self._open)
self.thread.setDaemon(True)
self.thread.start()
def _fn(reader, timeout=None):
delay = DELAY
timer = float(timeout or 0)
while not Thread.aborted():
message = fn(reader, timer)
if message:
return message
if timer > 0:
sleep(delay)
timer -= delay
if delay < MAX_DELAY:
delay *= DELAY_MULTIPLIER
else:
break
def _fn(reader, timeout=None):
delay = DELAY
timer = float(timeout or 0)
while not Thread.aborted():
message = fn(reader, timer)
if message:
return message
if timer > 0:
sleep(delay)
timer -= delay
if delay < MAX_DELAY:
delay *= DELAY_MULTIPLIER
else:
break
def _fn(messenger, *args, **kwargs):
repair = lambda: None
while not Thread.aborted():
try:
repair()
return fn(messenger, *args, **kwargs)
except ChannelError, e:
if e.code != 404:
sleep(DELAY)
repair = messenger.repair
else:
raise NotFound(*e.args)
except CONNECTION_EXCEPTIONS:
sleep(DELAY)
repair = messenger.repair
def _fn(messenger, *args, **kwargs):
repair = lambda: None
while not Thread.aborted():
try:
repair()
return fn(messenger, *args, **kwargs)
except LinkDetached, le:
if le.condition != NOT_FOUND:
sleep(DELAY)
repair = messenger.repair
else:
raise NotFound(*le.args)
except ConnectionException:
sleep(DELAY)
repair = messenger.repair
def _fn(messenger, *args, **kwargs):
repair = lambda: None
while not Thread.aborted():
try:
repair()
return fn(messenger, *args, **kwargs)
except ChannelError, e:
if e.code != 404:
sleep(DELAY)
repair = messenger.repair
else:
raise NotFound(*e.args)
except CONNECTION_EXCEPTIONS:
sleep(DELAY)
repair = messenger.repair
def _fn(messenger, *args, **kwargs):
repair = lambda: None
while not Thread.aborted():
try:
repair()
return fn(messenger, *args, **kwargs)
except LinkDetached, le:
if le.condition != NOT_FOUND:
sleep(DELAY)
repair = messenger.repair
else:
raise NotFound(*le.args)
except ConnectionException:
sleep(DELAY)
repair = messenger.repair
def inner(connection):
if connection.retry:
retries = RETRIES
else:
retries = 0
delay = DELAY
url = connection.url
while not Thread.aborted():
try:
log.info('connecting: %s', url)
impl = fn(connection)
log.info('connected: %s', url)
return impl
except exception, e:
log.error('connect: %s, failed: %s', url, e)
if retries > 0:
log.info('retry in %d seconds', delay)
sleep(delay)
if delay < MAX_DELAY:
delay *= DELAY_MULTIPLIER
retries -= 1
else:
raise
def inner(connection):
if connection.retry:
retries = RETRIES
else:
retries = 0
delay = DELAY
url = connection.url
while not Thread.aborted():
try:
log.info('connecting: %s', url)
impl = fn(connection)
log.info('connected: %s', url)
return impl
except exception, e:
log.error('connect: %s, failed: %s', url, e)
if retries > 0:
log.info('retry in %d seconds', delay)
sleep(delay)
if delay < MAX_DELAY:
delay *= DELAY_MULTIPLIER
retries -= 1
else:
raise
class Pending(object):
"""
Persistent store and queuing for pending requests.
"""
PENDING = '/var/lib/%s/messaging/pending' % NAME
@staticmethod
def _write(request, path):
"""
Write a request to the journal.
:param request: An AMQP request.
:type request: Document
:param path: The destination path.
:type path: str
"""
fp = open(path, 'w+')
try:
body = request.dump()
fp.write(body)
log.debug('wrote [%s]: %s', path, body)
finally:
fp.close()
@staticmethod
def _read(path):
"""
Read a request.
:param path: The path to the journal file.
:type path: str
:return: The read request.
:rtype: Document
"""
fp = open(path)
try:
try:
request = Document()
body = fp.read()
request.load(body)
log.debug('read [%s]: %s', path, body)
return request
except ValueError:
log.error('%s corrupt (discarded)', path)
os.unlink(path)
finally:
fp.close()
def _list(self):
"""
Directory listing sorted by when it was created.
:return: A sorted directory listing (absolute paths).
:rtype: list
"""
path = os.path.join(Pending.PENDING, self.stream)
paths = [os.path.join(path, name) for name in os.listdir(path)]
return sorted(paths)
def __init__(self, stream):
"""
:param stream: The stream name.
:type stream: str
"""
self.stream = stream
self.queue = Queue(maxsize=100)
self.is_open = False
self.sequential = Sequential()
self.journal = {}
self.thread = Thread(target=self._open)
self.thread.setDaemon(True)
self.thread.start()
def _open(self):
"""
Open for operations.
Load journal(ed) requests. These are requests were in the queuing pipeline
when the process was terminated. put() is blocked until this has completed.
"""
path = os.path.join(Pending.PENDING, self.stream)
mkdir(path)
log.info('Using: %s', path)
for path in self._list():
log.info('Restoring: %s', path)
request = Pending._read(path)
if not request:
# read failed
continue
self._put(request, path)
self.is_open = True
def put(self, request):
"""
Enqueue a pending request.
This is blocked until the _open() has re-queued journal(ed) entries.
:param request: An AMQP request.
:type request: Document
"""
while not self.is_open:
# block until opened
sleep(1)
fn = self.sequential.next()
path = os.path.join(Pending.PENDING, self.stream, fn)
Pending._write(request, path)
self._put(request, path)
def get(self):
"""
Get the next pending request to be dispatched.
Blocks until a request is available.
:return: The next pending request.
:rtype: Document
"""
while not Thread.aborted():
try:
return self.queue.get(timeout=10)
except Empty:
pass
def commit(self, sn):
"""
The request referenced by the serial number has been completely
processed and can be deleted from the journal.
:param sn: A request serial number.
:param sn: str
"""
try:
path = self.journal[sn]
unlink(path)
log.debug('%s committed', sn)
except KeyError:
log.warn('%s not found for commit', sn)
def delete(self):
"""
Drain the queue and delete the store.
"""
self.is_open = False
self.thread.abort()
self.thread.join()
self._drain()
path = os.path.join(Pending.PENDING, self.stream)
rmdir(path)
log.info('%s, deleted', path)
def _drain(self):
"""
Drain the queue.
"""
self.is_open = False
while not Thread.aborted():
try:
request = self.queue.get(timeout=1)
self.commit(request.sn)
except Empty:
break
def _put(self, request, jnl_path):
"""
Enqueue the request.
:param request: An AMQP request.
:type request: Document
:param jnl_path: Path to the associated journal file.
:type jnl_path: str
"""
request.ts = time()
tracker = Tracker()
tracker.add(request.sn, request.data)
self.journal[request.sn] = jnl_path
self.queue.put(request)
class Pending(object):
"""
Persistent store and queuing for pending requests.
"""
PENDING = '/var/lib/%s/messaging/pending' % NAME
@staticmethod
def _write(request, path):
"""
Write a request to the journal.
:param request: An AMQP request.
:type request: Document
:param path: The destination path.
:type path: str
"""
with open(path, 'w+') as fp:
body = request.dump()
fp.write(body)
log.debug('wrote [%s]: %s', path, body)
@staticmethod
def _read(path):
"""
Read a request.
:param path: The path to the journal file.
:type path: str
:return: The read request.
:rtype: Document
"""
with open(path) as fp:
try:
request = Document()
body = fp.read()
request.load(body)
log.debug('read [%s]: %s', path, body)
return request
except ValueError:
log.error('%s corrupt (discarded)', path)
unlink(path)
def _list(self):
"""
Directory listing sorted by when it was created.
:return: A sorted directory listing (absolute paths).
:rtype: list
"""
path = os.path.join(Pending.PENDING, self.stream)
paths = [os.path.join(path, name) for name in os.listdir(path)]
return sorted(paths)
def __init__(self, stream):
"""
:param stream: The stream name.
:type stream: str
"""
self.stream = stream
self.queue = Queue(maxsize=100)
self.is_open = False
self.sequential = Sequential()
self.journal = {}
self.thread = Thread(target=self._open)
self.thread.setDaemon(True)
self.thread.start()
def _open(self):
"""
Open for operations.
Load journal(ed) requests. These are requests were in the queuing pipeline
when the process was terminated. put() is blocked until this has completed.
"""
path = os.path.join(Pending.PENDING, self.stream)
mkdir(path)
log.info('Using: %s', path)
for path in self._list():
log.info('Restoring: %s', path)
request = Pending._read(path)
if not request:
# read failed
continue
self._put(request, path)
self.is_open = True
def put(self, request):
"""
Enqueue a pending request.
This is blocked until the _open() has re-queued journal(ed) entries.
:param request: An AMQP request.
:type request: Document
"""
while not self.is_open:
# block until opened
sleep(1)
fn = self.sequential.next()
path = os.path.join(Pending.PENDING, self.stream, fn)
Pending._write(request, path)
self._put(request, path)
def get(self):
"""
Get the next pending request to be dispatched.
Blocks until a request is available.
:return: The next pending request.
:rtype: Document
:raise Empty: on thread aborted.
"""
while not Thread.aborted():
try:
return self.queue.get(timeout=3)
except Empty:
pass
# aborted
raise Empty()
def commit(self, sn):
"""
The request referenced by the serial number has been completely
processed and can be deleted from the journal.
:param sn: A request serial number.
:param sn: str
"""
try:
path = self.journal.pop(sn)
unlink(path)
log.debug('%s committed', sn)
except KeyError:
log.warning('%s not found for commit', sn)
def delete(self):
"""
Drain the queue and delete the store.
"""
self.is_open = False
self.thread.abort()
self.thread.join()
self._drain()
path = os.path.join(Pending.PENDING, self.stream)
rmdir(path)
log.info('%s, deleted', path)
def _drain(self):
"""
Drain the queue.
"""
self.is_open = False
while not Thread.aborted():
try:
request = self.queue.get(timeout=1)
self.commit(request.sn)
except Empty:
break
def _put(self, request, jnl_path):
"""
Enqueue the request.
:param request: An AMQP request.
:type request: Document
:param jnl_path: Path to the associated journal file.
:type jnl_path: str
"""
request.ts = time()
tracker = Tracker()
tracker.add(request.sn, request.data)
self.journal[request.sn] = jnl_path
self.queue.put(request)