def get(self, block=True, timeout=None):
if not block:
return self.get_nowait()
self._consume()
time_start = monotonic()
remaining = timeout
while True:
if self.buffer:
return self.buffer.popleft()
if remaining is not None and remaining <= 0.0:
raise self.Empty()
try:
# The `drain_events` method will
# block on the socket connection to rabbitmq. if any
# application-level messages are received, it will put them
# into `self.buffer`.
# * The method will block for UP TO `timeout` milliseconds.
# * The method may raise a socket.timeout exception; or...
# * The method may return without having put anything on
# `self.buffer`. This is because internal heartbeat
# messages are sent over the same socket; also POSIX makes
# no guarantees against socket calls returning early.
self.channel.connection.client.drain_events(timeout=remaining)
except socket.timeout:
raise self.Empty()
if remaining is not None:
elapsed = monotonic() - time_start
remaining = timeout - elapsed
def drain_events(self, connection, timeout=None):
loop = 0
time_start = monotonic()
get = self.cycle.get
polling_interval = self.polling_interval
while 1:
try:
item, channel = get(timeout=timeout)
except Empty:
if timeout and monotonic() - time_start >= timeout:
raise socket.timeout()
loop += 1
if polling_interval is not None:
sleep(polling_interval)
else:
break
message, queue = item
if not queue or queue not in self._callbacks:
raise KeyError(
'Message for queue {0!r} without consumers: {1}'.format(
queue, message))
self._callbacks[queue](message)
def drain_events(self, connection, timeout=None):
loop = 0
time_start = monotonic()
get = self.cycle.get
polling_interval = self.polling_interval
while 1:
try:
item, channel = get(timeout=timeout)
except Empty:
if timeout and monotonic() - time_start >= timeout:
raise socket.timeout()
loop += 1
if polling_interval is not None:
sleep(polling_interval)
else:
break
message, queue = item
if not queue or queue not in self._callbacks:
raise KeyError(
'Message for queue {0!r} without consumers: {1}'.format(
queue, message))
self._callbacks[queue](message)
def bench_apply(n=DEFAULT_ITS):
time_start = monotonic()
task = it._get_current_object()
with app.producer_or_acquire() as producer:
[task.apply_async((i, n), producer=producer) for i in range(n)]
#group(s(i, n) for i in range(n))()
print('-- apply {0} tasks: {1}s'.format(n, monotonic() - time_start))
def drain_events(self, connection, timeout=None):
time_start = monotonic()
get = self.cycle.get
polling_interval = self.polling_interval
if timeout and polling_interval and polling_interval > timeout:
polling_interval = timeout
consume_pending = False # We know there is a message to consume because we've been
while 1:
try:
get(self._deliver, timeout=timeout)
consume_pending = False
except Empty:
if self.shutdown or (timeout is not None
and monotonic() - time_start >= timeout):
raise socket.timeout()
if consume_pending:
sleep(0.000001)
elif polling_interval is not None:
notifier = TRANSPORT_NOTIFIERS[self]
was_notified = notifier.wait(polling_interval)
if not was_notified and timeout is not None and monotonic(
) - time_start >= timeout:
raise socket.timeout()
notifier.clear()
consume_pending = True
sleep(0.000001)
else:
break
def drain_events(self, connection, timeout=None):
time_start = monotonic()
get = self.cycle.get
polling_interval = self.polling_interval
while 1:
try:
get(self._deliver, timeout=timeout)
except Empty:
if timeout is not None and monotonic() - time_start >= timeout:
raise socket.timeout()
if polling_interval is not None:
sleep(polling_interval)
else:
break
def drain_events(self, connection, timeout=None):
time_start = monotonic()
get = self.cycle.get
polling_interval = self.polling_interval
while 1:
try:
get(self._deliver, timeout=timeout)
except Empty:
if timeout is not None and monotonic() - time_start >= timeout:
raise socket.timeout()
if polling_interval is not None:
sleep(polling_interval)
else:
break
def memcache_lock(lock_id, oid):
timeout_at = monotonic() + LOCK_EXPIRE - 3
# cache.add fails if the key already exists
status = cache.add(lock_id, oid, LOCK_EXPIRE)
try:
yield status
finally:
# memcache delete is very slow, but we have to use it to take
# advantage of using add() for atomic locking
if monotonic() < timeout_at and status:
# don't release the lock if we exceeded the timeout
# to lessen the chance of releasing an expired lock
# owned by someone else
# also don't release the lock if we didn't acquire it
cache.delete(lock_id)
def _get_tokens(self):
if self._tokens < self.capacity:
now = monotonic()
delta = self.fill_rate * (now - self.timestamp)
self._tokens = min(self.capacity, self._tokens + delta)
self.timestamp = now
return self._tokens
def _get_tokens(self):
if self._tokens < self.capacity:
now = monotonic()
delta = self.fill_rate * (now - self.timestamp)
self._tokens = min(self.capacity, self._tokens + delta)
self.timestamp = now
return self._tokens
def drain_events(self, connection, timeout=None):
loop = 0
time_start = monotonic()
get = self.cycle.get
polling_interval = self.polling_interval
while 1:
try:
item, channel = get(timeout=timeout)
except Empty:
if timeout and monotonic() - time_start >= timeout:
raise socket.timeout()
loop += 1
if polling_interval is not None:
sleep(polling_interval)
else:
break
self._deliver(*item)
def it(_, n):
# use internal counter, as ordering can be skewed
# by previous runs, or the broker.
i = it.cur
if i and not i % 5000:
print("({0} so far: {1}s)".format(i, tdiff(it.subt)), file=sys.stderr)
it.subt = monotonic()
if not i:
it.subt = it.time_start = monotonic()
elif i > n - 2:
total = tdiff(it.time_start)
print("({0} so far: {1}s)".format(i, tdiff(it.subt)), file=sys.stderr)
print("-- process {0} tasks: {1}s total, {2} tasks/s} ".format(n, total, n / (total + 0.0)))
import os
os._exit()
it.cur += 1
def drain_events(self, connection, timeout=None):
loop = 0
time_start = monotonic()
get = self.cycle.get
polling_interval = self.polling_interval
while 1:
try:
item, channel = get(timeout=timeout)
except Empty:
if timeout and monotonic() - time_start >= timeout:
raise socket.timeout()
loop += 1
if polling_interval is not None:
sleep(polling_interval)
else:
break
self._deliver(*item)
def get(self, block=True, timeout=None):
if not block:
return self.get_nowait()
self._consume()
elapsed = 0.0
remaining = timeout
while True:
time_start = monotonic()
if self.buffer:
return self.buffer.popleft()
try:
self.channel.connection.client.drain_events(
timeout=timeout and remaining)
except socket.timeout:
raise Empty()
elapsed += monotonic() - time_start
remaining = timeout and timeout - elapsed or None
def get(self, block=True, timeout=None):
if not block:
return self.get_nowait()
self._consume()
elapsed = 0.0
remaining = timeout
while True:
time_start = monotonic()
if self.buffer:
return self.buffer.popleft()
try:
self.channel.connection.client.drain_events(
timeout=timeout and remaining)
except socket.timeout:
raise Empty()
elapsed += monotonic() - time_start
remaining = timeout and timeout - elapsed or None
def it(_, n):
# use internal counter, as ordering can be skewed
# by previous runs, or the broker.
i = it.cur
if i and not i % 5000:
print('({0} so far: {1}s)'.format(i, tdiff(it.subt)), file=sys.stderr)
it.subt = monotonic()
if not i:
it.subt = it.time_start = monotonic()
elif i > n - 2:
total = tdiff(it.time_start)
print('({0} so far: {1}s)'.format(i, tdiff(it.subt)), file=sys.stderr)
print('-- process {0} tasks: {1}s total, {2} tasks/s'.format(
n, total, n / (total + .0),
))
import os
os._exit()
it.cur += 1
def home(request):
# {u'celery@blackpearl': [
# {
# u'args': u'[]',
# u'time_start': 37030.852239337,
# u'name': u'crawler_server.tasks.RunSpider',
# u'delivery_info': {
# u'priority': None,
# u'redelivered': False,
# u'routing_key': u'celery',
# u'exchange': u'celery'
# },
# u'hostname': u'celery@blackpearl',
# u'acknowledged': True,
# u'kwargs': u"{u'domain': u'localhost', u'starturl': u'http://localhost/bizlist/'}",
# u'id': u'a5fcab29-376a-4cfe-9281-746e99bb100d',
# u'worker_pid': 23993
# }
# ]
# }
i = inspect()
raw_active = i.active().values()[0]
active = [{
'id': act['id'],
'pid': act['worker_pid'],
'start_time': datetime.fromtimestamp(time() - (monotonic() - act['time_start'])),
'domain': act['kwargs'],
'runtime': datetime.now() - (datetime.fromtimestamp(time() - (monotonic() - act['time_start']))),
'logurl': '/static/logs/' + act['id'] + '.log',
'size':get_logfilesize(act['id'])
} for act in raw_active]
# act = raw_active[0]
# active = {}
# active['id'] = act['id']
# active['pid'] = act['worker_pid']
# active['start_time'] = datetime.fromtimestamp(time() - (monotonic() - act['time_start']))
# active['domain'] = act['kwargs']
# active['runtime'] = datetime.now() - active['start_time']
# active['logurl'] = '/static/logs/'+act['id']+'.log'
scheduled = []
finished = get_finished()
data = {'scheduled': scheduled, 'active': active, 'finished': finished}
return render_to_response('johnnywalker/home.html', data)
def _enter(self, eta, priority, entry, **kwargs):
secs = max(eta - monotonic(), 0)
g = self._spawn_after(secs, entry)
self._queue.add(g)
g.link(self._entry_exit, entry)
g.entry = entry
g.eta = eta
g.priority = priority
g.canceled = False
return g
def _enter(self, eta, priority, entry):
secs = max(eta - monotonic(), 0)
g = self._Greenlet.spawn_later(secs, entry)
self._queue.add(g)
g.link(self._entry_exit)
g.entry = entry
g.eta = eta
g.priority = priority
g.canceled = False
return g
def _enter(self, eta, priority, entry):
secs = max(eta - monotonic(), 0)
g = self._Greenlet.spawn_later(secs, entry)
self._queue.add(g)
g.link(self._entry_exit)
g.entry = entry
g.eta = eta
g.priority = priority
g.canceled = False
return g
def _enter(self, eta, priority, entry, **kwargs):
secs = max(eta - monotonic(), 0)
g = self._spawn_after(secs, entry)
self._queue.add(g)
g.link(self._entry_exit, entry)
g.entry = entry
g.eta = eta
g.priority = priority
g.canceled = False
return g
def _reschedules(*args, **kwargs):
last, now = tref._last_run, monotonic()
lsince = (now - tref._last_run) if last else secs
try:
if lsince and lsince >= secs:
tref._last_run = now
return fun(*args, **kwargs)
finally:
if not tref.canceled:
last = tref._last_run
next = secs - (now - last) if last else secs
self.enter_after(next, tref, priority)
def _put(self, queue, payload, **_):
"""Put `message` onto `queue`.
This simply writes a key to the K/V store of Consul
"""
key = '{0}/msg/{1}_{2}'.format(
self._key_prefix(queue),
int(round(monotonic() * 1000)),
uuid.uuid4(),
)
if not self.client.kv.put(key=key, value=dumps(payload), cas=0):
raise ChannelError('Cannot add key {0!r} to consul'.format(key))
def _put(self, queue, payload, **_):
"""Put `message` onto `queue`.
This simply writes a key to the K/V store of Consul
"""
key = '{0}/msg/{1}_{2}'.format(
self._key_prefix(queue),
int(round(monotonic() * 1000)),
uuid.uuid4(),
)
if not self.client.kv.put(key=key, value=dumps(payload), cas=0):
raise ChannelError('Cannot add key {0!r} to consul'.format(key))
def _reschedules(*args, **kwargs):
last, now = tref._last_run, monotonic()
lsince = (now - tref._last_run) if last else secs
try:
if lsince and lsince >= secs:
tref._last_run = now
return fun(*args, **kwargs)
finally:
if not tref.cancelled:
last = tref._last_run
next = secs - (now - last) if last else secs
self.enter_after(next, tref, priority)
def _put(self, queue, payload, **kwargs):
"""Put `message` onto `queue`."""
filename = "%s_%s.%s.msg" % (int(round(monotonic() * 1000)), uuid.uuid4(), queue)
filename = os.path.join(self.data_folder_out, filename)
try:
f = open(filename, "wb")
lock(f, LOCK_EX)
f.write(str_to_bytes(dumps(payload)))
except (IOError, OSError):
raise ChannelError("Cannot add file {0!r} to directory".format(filename))
finally:
unlock(f)
f.close()
def _put(self, queue, payload, **kwargs):
"""Put `message` onto `queue`."""
filename = '%s_%s.%s.msg' % (int(round(
monotonic() * 1000)), uuid.uuid4(), queue)
filename = os.path.join(self.data_folder_out, filename)
try:
f = open(filename, 'wb')
lock(f, LOCK_EX)
f.write(str_to_bytes(dumps(payload)))
except (IOError, OSError):
raise ChannelError(
'Cannot add file {0!r} to directory'.format(filename))
finally:
unlock(f)
f.close()
def tdiff(then):
return monotonic() - then
def test_timeout_over_polling_interval(self):
x = client(transport_options=dict(polling_interval=60))
start = monotonic()
with pytest.raises(socket.timeout):
x.transport.drain_events(x, timeout=.5)
assert monotonic() - start < 60
def tdiff(then):
return monotonic() - then
def __init__(self, fill_rate, capacity=1):
self.capacity = float(capacity)
self._tokens = capacity
self.fill_rate = float(fill_rate)
self.timestamp = monotonic()
def __init__(self, fill_rate, capacity=1):
self.capacity = float(capacity)
self._tokens = capacity
self.fill_rate = float(fill_rate)
self.timestamp = monotonic()
self.contents = deque()
