def get(self, block=True, timeout=None): if not block: return self.get_nowait() self._consume() time_start = monotonic() remaining = timeout while True: if self.buffer: return self.buffer.popleft() if remaining is not None and remaining <= 0.0: raise self.Empty() try: # The `drain_events` method will # block on the socket connection to rabbitmq. if any # application-level messages are received, it will put them # into `self.buffer`. # * The method will block for UP TO `timeout` milliseconds. # * The method may raise a socket.timeout exception; or... # * The method may return without having put anything on # `self.buffer`. This is because internal heartbeat # messages are sent over the same socket; also POSIX makes # no guarantees against socket calls returning early. self.channel.connection.client.drain_events(timeout=remaining) except socket.timeout: raise self.Empty() if remaining is not None: elapsed = monotonic() - time_start remaining = timeout - elapsed
def drain_events(self, connection, timeout=None): loop = 0 time_start = monotonic() get = self.cycle.get polling_interval = self.polling_interval while 1: try: item, channel = get(timeout=timeout) except Empty: if timeout and monotonic() - time_start >= timeout: raise socket.timeout() loop += 1 if polling_interval is not None: sleep(polling_interval) else: break message, queue = item if not queue or queue not in self._callbacks: raise KeyError( 'Message for queue {0!r} without consumers: {1}'.format( queue, message)) self._callbacks[queue](message)
def bench_apply(n=DEFAULT_ITS): time_start = monotonic() task = it._get_current_object() with app.producer_or_acquire() as producer: [task.apply_async((i, n), producer=producer) for i in range(n)] #group(s(i, n) for i in range(n))() print('-- apply {0} tasks: {1}s'.format(n, monotonic() - time_start))
def drain_events(self, connection, timeout=None): time_start = monotonic() get = self.cycle.get polling_interval = self.polling_interval if timeout and polling_interval and polling_interval > timeout: polling_interval = timeout consume_pending = False # We know there is a message to consume because we've been while 1: try: get(self._deliver, timeout=timeout) consume_pending = False except Empty: if self.shutdown or (timeout is not None and monotonic() - time_start >= timeout): raise socket.timeout() if consume_pending: sleep(0.000001) elif polling_interval is not None: notifier = TRANSPORT_NOTIFIERS[self] was_notified = notifier.wait(polling_interval) if not was_notified and timeout is not None and monotonic( ) - time_start >= timeout: raise socket.timeout() notifier.clear() consume_pending = True sleep(0.000001) else: break
def drain_events(self, connection, timeout=None): time_start = monotonic() get = self.cycle.get polling_interval = self.polling_interval while 1: try: get(self._deliver, timeout=timeout) except Empty: if timeout is not None and monotonic() - time_start >= timeout: raise socket.timeout() if polling_interval is not None: sleep(polling_interval) else: break
def memcache_lock(lock_id, oid): timeout_at = monotonic() + LOCK_EXPIRE - 3 # cache.add fails if the key already exists status = cache.add(lock_id, oid, LOCK_EXPIRE) try: yield status finally: # memcache delete is very slow, but we have to use it to take # advantage of using add() for atomic locking if monotonic() < timeout_at and status: # don't release the lock if we exceeded the timeout # to lessen the chance of releasing an expired lock # owned by someone else # also don't release the lock if we didn't acquire it cache.delete(lock_id)
def _get_tokens(self): if self._tokens < self.capacity: now = monotonic() delta = self.fill_rate * (now - self.timestamp) self._tokens = min(self.capacity, self._tokens + delta) self.timestamp = now return self._tokens
def drain_events(self, connection, timeout=None): loop = 0 time_start = monotonic() get = self.cycle.get polling_interval = self.polling_interval while 1: try: item, channel = get(timeout=timeout) except Empty: if timeout and monotonic() - time_start >= timeout: raise socket.timeout() loop += 1 if polling_interval is not None: sleep(polling_interval) else: break self._deliver(*item)
def it(_, n): # use internal counter, as ordering can be skewed # by previous runs, or the broker. i = it.cur if i and not i % 5000: print("({0} so far: {1}s)".format(i, tdiff(it.subt)), file=sys.stderr) it.subt = monotonic() if not i: it.subt = it.time_start = monotonic() elif i > n - 2: total = tdiff(it.time_start) print("({0} so far: {1}s)".format(i, tdiff(it.subt)), file=sys.stderr) print("-- process {0} tasks: {1}s total, {2} tasks/s} ".format(n, total, n / (total + 0.0))) import os os._exit() it.cur += 1
def get(self, block=True, timeout=None): if not block: return self.get_nowait() self._consume() elapsed = 0.0 remaining = timeout while True: time_start = monotonic() if self.buffer: return self.buffer.popleft() try: self.channel.connection.client.drain_events( timeout=timeout and remaining) except socket.timeout: raise Empty() elapsed += monotonic() - time_start remaining = timeout and timeout - elapsed or None
def it(_, n): # use internal counter, as ordering can be skewed # by previous runs, or the broker. i = it.cur if i and not i % 5000: print('({0} so far: {1}s)'.format(i, tdiff(it.subt)), file=sys.stderr) it.subt = monotonic() if not i: it.subt = it.time_start = monotonic() elif i > n - 2: total = tdiff(it.time_start) print('({0} so far: {1}s)'.format(i, tdiff(it.subt)), file=sys.stderr) print('-- process {0} tasks: {1}s total, {2} tasks/s'.format( n, total, n / (total + .0), )) import os os._exit() it.cur += 1
def home(request): # {u'celery@blackpearl': [ # { # u'args': u'[]', # u'time_start': 37030.852239337, # u'name': u'crawler_server.tasks.RunSpider', # u'delivery_info': { # u'priority': None, # u'redelivered': False, # u'routing_key': u'celery', # u'exchange': u'celery' # }, # u'hostname': u'celery@blackpearl', # u'acknowledged': True, # u'kwargs': u"{u'domain': u'localhost', u'starturl': u'http://localhost/bizlist/'}", # u'id': u'a5fcab29-376a-4cfe-9281-746e99bb100d', # u'worker_pid': 23993 # } # ] # } i = inspect() raw_active = i.active().values()[0] active = [{ 'id': act['id'], 'pid': act['worker_pid'], 'start_time': datetime.fromtimestamp(time() - (monotonic() - act['time_start'])), 'domain': act['kwargs'], 'runtime': datetime.now() - (datetime.fromtimestamp(time() - (monotonic() - act['time_start']))), 'logurl': '/static/logs/' + act['id'] + '.log', 'size':get_logfilesize(act['id']) } for act in raw_active] # act = raw_active[0] # active = {} # active['id'] = act['id'] # active['pid'] = act['worker_pid'] # active['start_time'] = datetime.fromtimestamp(time() - (monotonic() - act['time_start'])) # active['domain'] = act['kwargs'] # active['runtime'] = datetime.now() - active['start_time'] # active['logurl'] = '/static/logs/'+act['id']+'.log' scheduled = [] finished = get_finished() data = {'scheduled': scheduled, 'active': active, 'finished': finished} return render_to_response('johnnywalker/home.html', data)
def _enter(self, eta, priority, entry, **kwargs): secs = max(eta - monotonic(), 0) g = self._spawn_after(secs, entry) self._queue.add(g) g.link(self._entry_exit, entry) g.entry = entry g.eta = eta g.priority = priority g.canceled = False return g
def _enter(self, eta, priority, entry): secs = max(eta - monotonic(), 0) g = self._Greenlet.spawn_later(secs, entry) self._queue.add(g) g.link(self._entry_exit) g.entry = entry g.eta = eta g.priority = priority g.canceled = False return g
def _reschedules(*args, **kwargs): last, now = tref._last_run, monotonic() lsince = (now - tref._last_run) if last else secs try: if lsince and lsince >= secs: tref._last_run = now return fun(*args, **kwargs) finally: if not tref.canceled: last = tref._last_run next = secs - (now - last) if last else secs self.enter_after(next, tref, priority)
def _put(self, queue, payload, **_): """Put `message` onto `queue`. This simply writes a key to the K/V store of Consul """ key = '{0}/msg/{1}_{2}'.format( self._key_prefix(queue), int(round(monotonic() * 1000)), uuid.uuid4(), ) if not self.client.kv.put(key=key, value=dumps(payload), cas=0): raise ChannelError('Cannot add key {0!r} to consul'.format(key))
def _reschedules(*args, **kwargs): last, now = tref._last_run, monotonic() lsince = (now - tref._last_run) if last else secs try: if lsince and lsince >= secs: tref._last_run = now return fun(*args, **kwargs) finally: if not tref.cancelled: last = tref._last_run next = secs - (now - last) if last else secs self.enter_after(next, tref, priority)
def _put(self, queue, payload, **kwargs): """Put `message` onto `queue`.""" filename = "%s_%s.%s.msg" % (int(round(monotonic() * 1000)), uuid.uuid4(), queue) filename = os.path.join(self.data_folder_out, filename) try: f = open(filename, "wb") lock(f, LOCK_EX) f.write(str_to_bytes(dumps(payload))) except (IOError, OSError): raise ChannelError("Cannot add file {0!r} to directory".format(filename)) finally: unlock(f) f.close()
def _put(self, queue, payload, **kwargs): """Put `message` onto `queue`.""" filename = '%s_%s.%s.msg' % (int(round( monotonic() * 1000)), uuid.uuid4(), queue) filename = os.path.join(self.data_folder_out, filename) try: f = open(filename, 'wb') lock(f, LOCK_EX) f.write(str_to_bytes(dumps(payload))) except (IOError, OSError): raise ChannelError( 'Cannot add file {0!r} to directory'.format(filename)) finally: unlock(f) f.close()
def tdiff(then): return monotonic() - then
def test_timeout_over_polling_interval(self): x = client(transport_options=dict(polling_interval=60)) start = monotonic() with pytest.raises(socket.timeout): x.transport.drain_events(x, timeout=.5) assert monotonic() - start < 60
def __init__(self, fill_rate, capacity=1): self.capacity = float(capacity) self._tokens = capacity self.fill_rate = float(fill_rate) self.timestamp = monotonic()
def __init__(self, fill_rate, capacity=1): self.capacity = float(capacity) self._tokens = capacity self.fill_rate = float(fill_rate) self.timestamp = monotonic() self.contents = deque()