def push(self, consistency, application_name, queue_name, message, metadata=None, ttl=60 * 60 * 24 * 3, timestamp=None): """Push a message onto the queue""" cl = self.cl or self._get_cl(consistency) if not timestamp: now = uuid.uuid1() elif isinstance(timestamp, (float, Decimal)): now = convert_time_to_uuid(timestamp, randomize=True) else: now = uuid.UUID(hex=timestamp) queue_name = '%s:%s' % (application_name, queue_name) if metadata: batch = pycassa.batch.Mutator(self.pool, write_consistency_level=cl) batch.insert(self.message_fam, key=queue_name, columns={now: message}, ttl=ttl) batch.insert(self.meta_fam, key=now, columns=metadata, ttl=ttl) batch.send() else: self.message_fam.insert(key=queue_name, columns={now: message}, ttl=ttl, write_consistency_level=cl) timestamp = Decimal(now.time - 0x01b21dd213814000L) / DECIMAL_1E7 return now.hex, timestamp
def retrieve_batch(self, consistency, application_name, queue_names, limit=None, include_metadata=False, start_at=None, order="ascending"): """Retrieve a batch of messages off the queue""" if not isinstance(queue_names, list): raise Exception("queue_names must be a list") cl = self.cl or self._get_cl(consistency) delay = self._get_delay(consistency) kwargs = {'read_consistency_level': cl} if order == 'descending': kwargs['column_reversed'] = True if limit: kwargs['column_count'] = limit if start_at: if isinstance(start_at, basestring): # Assume its a hex, transform to a datetime start_at = uuid.UUID(hex=start_at) else: # Assume its a float/decimal, convert to UUID start_at = convert_time_to_uuid(start_at) kwargs['column_start'] = start_at queue_names = ['%s:%s' % (application_name, x) for x in queue_names] results = self.message_fam.multiget(keys=queue_names, **kwargs) results = results.items() if delay: cut_off = time.time() - delay # Turn it into time in ns, for efficient comparison cut_off = int(cut_off * 1e7) + 0x01b21dd213814000L result_list = [] msg_hash = {} for queue_name, messages in results: for msg_id, body in messages.items(): if delay and msg_id.time >= cut_off: continue obj = { 'message_id': msg_id.hex, 'timestamp': (Decimal(msg_id.time - 0x01b21dd213814000L) / DECIMAL_1E7), 'body': body, 'metadata': {}, 'queue_name': queue_name[queue_name.find(':'):] } result_list.append(obj) msg_hash[msg_id] = obj # Get metadata? if include_metadata: results = self.meta_fam.multiget(keys=msg_hash.keys()) for msg_id, metadata in results.items(): msg_hash[msg_id]['metadata'] = metadata return result_list
def push(self, consistency, application_name, queue_name, message, metadata=None, ttl=60 * 60 * 24 * 3, timestamp=None): """Push a message onto the queue""" if not timestamp: now = uuid.uuid1() elif isinstance(timestamp, (float, Decimal)): now = convert_time_to_uuid(timestamp, randomize=True) else: now = uuid.UUID(hex=timestamp) msg = Message(id=now, body=message, ttl=ttl) if metadata: msg.metadata = metadata timestamp = Decimal(msg.id.time - 0x01b21dd213814000L) / DECIMAL_1E7 queue_name = '%s:%s' % (application_name, queue_name) if msg in message_store[queue_name]: message_store[queue_name].remove(msg) message_store[queue_name].append(msg) return msg.id.hex, timestamp
def retrieve(self, consistency, application_name, queue_name, message_id, include_metadata=False): """Retrieve a single message""" if isinstance(message_id, basestring): # Convert to uuid for lookup message_id = uuid.UUID(hex=message_id) else: # Assume its a float/decimal, convert to UUID message_id = convert_time_to_uuid(message_id) queue_name = '%s:%s' % (application_name, queue_name) queue = message_store[queue_name] found = None for msg in queue: if msg.id == message_id: found = msg break if not found: return {} now = Decimal(repr(time.time())) if found.expiration and now > found.expiration: queue.remove(found) return {} obj = { 'message_id': found.id.hex, 'timestamp': (Decimal(found.id.time - 0x01b21dd213814000L) / DECIMAL_1E7), 'body': found.body, 'metadata': {}, 'queue_name': queue_name[queue_name.find(':'):] } if include_metadata: obj['metadata'] = found.metadata return obj
def retrieve(self, consistency, application_name, queue_name, message_id, include_metadata=False): """Retrieve a single message""" cl = self.cl or self._get_cl(consistency) if isinstance(message_id, basestring): # Convert to uuid for lookup message_id = uuid.UUID(hex=message_id) else: # Assume its a float/decimal, convert to UUID message_id = convert_time_to_uuid(message_id) kwargs = {'read_consistency_level': cl, 'columns': [message_id]} queue_name = '%s:%s' % (application_name, queue_name) try: results = self.message_fam.get(key=queue_name, **kwargs) except (pycassa.NotFoundException, pycassa.InvalidRequestException): return {} msg_id, body = results.items()[0] obj = { 'message_id': msg_id.hex, 'timestamp': (Decimal(msg_id.time - 0x01b21dd213814000L) / DECIMAL_1E7), 'body': body, 'metadata': {}, 'queue_name': queue_name[queue_name.find(':'):] } # Get metadata? if include_metadata: try: results = self.meta_fam.get(key=msg_id) obj['metadata'] = results except pycassa.NotFoundException: pass return obj
def retrieve(self, consistency, application_name, queue_name, message_id, include_metadata=False): """Retrieve a single message""" cl = self.cl or self._get_cl(consistency) if isinstance(message_id, basestring): # Convert to uuid for lookup message_id = uuid.UUID(hex=message_id) else: # Assume its a float/decimal, convert to UUID message_id = convert_time_to_uuid(message_id) kwargs = { 'read_consistency_level': cl, 'columns': [message_id]} queue_name = '%s:%s' % (application_name, queue_name) try: results = self.message_fam.get(key=queue_name, **kwargs) except (pycassa.NotFoundException, pycassa.InvalidRequestException): return {} msg_id, body = results.items()[0] obj = { 'message_id': msg_id.hex, 'timestamp': (Decimal(msg_id.time - 0x01b21dd213814000L) / DECIMAL_1E7), 'body': body, 'metadata': {}, 'queue_name': queue_name[queue_name.find(':'):] } # Get metadata? if include_metadata: try: results = self.meta_fam.get(key=msg_id) obj['metadata'] = results except pycassa.NotFoundException: pass return obj
def retrieve_batch(self, consistency, application_name, queue_names, limit=None, include_metadata=False, start_at=None, order="ascending"): """Retrieve a batch of messages off the queue""" if not isinstance(queue_names, list): raise Exception("queue_names must be a list") order = -1 if order == 'descending' else 1 if start_at: if isinstance(start_at, basestring): # Assume its a hex, transform to a UUID start_at = uuid.UUID(hex=start_at) else: # Assume its a float/decimal, convert to UUID start_at = convert_time_to_uuid(start_at) queue_names = ['%s:%s' % (application_name, x) for x in queue_names] results = [] now = Decimal(repr(time.time())) for queue_name in queue_names: msgs = message_store[queue_name] if not msgs: continue msgs.sort(key=lambda k: (k.id.time, k.id.bytes)) if start_at: # Locate the index given the start_at point = (start_at.time, start_at.bytes) beg = msgs[0].id end = msgs[-1].id if point <= (beg.time, beg.bytes): # Is the start_at less than the beginning? Start at beginning start = 0 elif point >= (end.time, end.bytes): # Is the start_at larger than the end? Start at the end start = len(msgs) - 1 else: # The start point is somewhere inside, skim through until # we hit a value a value equal to or greater than start = 0 msg_comp = (msgs[start].id.time, msgs[start].id.bytes) while point > msg_comp: start += 1 msg_comp = (msgs[start].id.time, msgs[start].id.bytes) else: if order == -1: start = len(msgs) - 1 else: start = 0 count = 0 for msg in msgs[start::order]: if msg.expiration and now > msg.expiration: msgs.remove(msg) continue count += 1 if limit and count > limit: break obj = { 'message_id': msg.id.hex, 'timestamp': (Decimal(msg.id.time - 0x01b21dd213814000L) / DECIMAL_1E7), 'body': msg.body, 'metadata': {}, 'queue_name': queue_name[queue_name.find(':'):] } if include_metadata: obj['metadata'] = msg.metadata results.append(obj) return results