def push(self,
consistency,
application_name,
queue_name,
message,
metadata=None,
ttl=60 * 60 * 24 * 3,
timestamp=None):
"""Push a message onto the queue"""
cl = self.cl or self._get_cl(consistency)
if not timestamp:
now = uuid.uuid1()
elif isinstance(timestamp, (float, Decimal)):
now = convert_time_to_uuid(timestamp, randomize=True)
else:
now = uuid.UUID(hex=timestamp)
queue_name = '%s:%s' % (application_name, queue_name)
if metadata:
batch = pycassa.batch.Mutator(self.pool,
write_consistency_level=cl)
batch.insert(self.message_fam,
key=queue_name,
columns={now: message},
ttl=ttl)
batch.insert(self.meta_fam, key=now, columns=metadata, ttl=ttl)
batch.send()
else:
self.message_fam.insert(key=queue_name,
columns={now: message},
ttl=ttl,
write_consistency_level=cl)
timestamp = Decimal(now.time - 0x01b21dd213814000L) / DECIMAL_1E7
return now.hex, timestamp
def retrieve_batch(self, consistency, application_name, queue_names,
limit=None, include_metadata=False, start_at=None,
order="ascending"):
"""Retrieve a batch of messages off the queue"""
if not isinstance(queue_names, list):
raise Exception("queue_names must be a list")
cl = self.cl or self._get_cl(consistency)
delay = self._get_delay(consistency)
kwargs = {'read_consistency_level': cl}
if order == 'descending':
kwargs['column_reversed'] = True
if limit:
kwargs['column_count'] = limit
if start_at:
if isinstance(start_at, basestring):
# Assume its a hex, transform to a datetime
start_at = uuid.UUID(hex=start_at)
else:
# Assume its a float/decimal, convert to UUID
start_at = convert_time_to_uuid(start_at)
kwargs['column_start'] = start_at
queue_names = ['%s:%s' % (application_name, x) for x in queue_names]
results = self.message_fam.multiget(keys=queue_names, **kwargs)
results = results.items()
if delay:
cut_off = time.time() - delay
# Turn it into time in ns, for efficient comparison
cut_off = int(cut_off * 1e7) + 0x01b21dd213814000L
result_list = []
msg_hash = {}
for queue_name, messages in results:
for msg_id, body in messages.items():
if delay and msg_id.time >= cut_off:
continue
obj = {
'message_id': msg_id.hex,
'timestamp': (Decimal(msg_id.time - 0x01b21dd213814000L) /
DECIMAL_1E7),
'body': body,
'metadata': {},
'queue_name': queue_name[queue_name.find(':'):]
}
result_list.append(obj)
msg_hash[msg_id] = obj
# Get metadata?
if include_metadata:
results = self.meta_fam.multiget(keys=msg_hash.keys())
for msg_id, metadata in results.items():
msg_hash[msg_id]['metadata'] = metadata
return result_list
def push(self, consistency, application_name, queue_name, message,
metadata=None, ttl=60 * 60 * 24 * 3, timestamp=None):
"""Push a message onto the queue"""
if not timestamp:
now = uuid.uuid1()
elif isinstance(timestamp, (float, Decimal)):
now = convert_time_to_uuid(timestamp, randomize=True)
else:
now = uuid.UUID(hex=timestamp)
msg = Message(id=now, body=message, ttl=ttl)
if metadata:
msg.metadata = metadata
timestamp = Decimal(msg.id.time - 0x01b21dd213814000L) / DECIMAL_1E7
queue_name = '%s:%s' % (application_name, queue_name)
if msg in message_store[queue_name]:
message_store[queue_name].remove(msg)
message_store[queue_name].append(msg)
return msg.id.hex, timestamp
def retrieve(self,
consistency,
application_name,
queue_name,
message_id,
include_metadata=False):
"""Retrieve a single message"""
if isinstance(message_id, basestring):
# Convert to uuid for lookup
message_id = uuid.UUID(hex=message_id)
else:
# Assume its a float/decimal, convert to UUID
message_id = convert_time_to_uuid(message_id)
queue_name = '%s:%s' % (application_name, queue_name)
queue = message_store[queue_name]
found = None
for msg in queue:
if msg.id == message_id:
found = msg
break
if not found:
return {}
now = Decimal(repr(time.time()))
if found.expiration and now > found.expiration:
queue.remove(found)
return {}
obj = {
'message_id':
found.id.hex,
'timestamp':
(Decimal(found.id.time - 0x01b21dd213814000L) / DECIMAL_1E7),
'body':
found.body,
'metadata': {},
'queue_name':
queue_name[queue_name.find(':'):]
}
if include_metadata:
obj['metadata'] = found.metadata
return obj
def retrieve(self,
consistency,
application_name,
queue_name,
message_id,
include_metadata=False):
"""Retrieve a single message"""
cl = self.cl or self._get_cl(consistency)
if isinstance(message_id, basestring):
# Convert to uuid for lookup
message_id = uuid.UUID(hex=message_id)
else:
# Assume its a float/decimal, convert to UUID
message_id = convert_time_to_uuid(message_id)
kwargs = {'read_consistency_level': cl, 'columns': [message_id]}
queue_name = '%s:%s' % (application_name, queue_name)
try:
results = self.message_fam.get(key=queue_name, **kwargs)
except (pycassa.NotFoundException, pycassa.InvalidRequestException):
return {}
msg_id, body = results.items()[0]
obj = {
'message_id': msg_id.hex,
'timestamp':
(Decimal(msg_id.time - 0x01b21dd213814000L) / DECIMAL_1E7),
'body': body,
'metadata': {},
'queue_name': queue_name[queue_name.find(':'):]
}
# Get metadata?
if include_metadata:
try:
results = self.meta_fam.get(key=msg_id)
obj['metadata'] = results
except pycassa.NotFoundException:
pass
return obj
def retrieve(self, consistency, application_name, queue_name, message_id,
include_metadata=False):
"""Retrieve a single message"""
cl = self.cl or self._get_cl(consistency)
if isinstance(message_id, basestring):
# Convert to uuid for lookup
message_id = uuid.UUID(hex=message_id)
else:
# Assume its a float/decimal, convert to UUID
message_id = convert_time_to_uuid(message_id)
kwargs = {
'read_consistency_level': cl,
'columns': [message_id]}
queue_name = '%s:%s' % (application_name, queue_name)
try:
results = self.message_fam.get(key=queue_name, **kwargs)
except (pycassa.NotFoundException, pycassa.InvalidRequestException):
return {}
msg_id, body = results.items()[0]
obj = {
'message_id': msg_id.hex,
'timestamp': (Decimal(msg_id.time - 0x01b21dd213814000L) /
DECIMAL_1E7),
'body': body,
'metadata': {},
'queue_name': queue_name[queue_name.find(':'):]
}
# Get metadata?
if include_metadata:
try:
results = self.meta_fam.get(key=msg_id)
obj['metadata'] = results
except pycassa.NotFoundException:
pass
return obj
def push(self, consistency, application_name, queue_name, message,
metadata=None, ttl=60 * 60 * 24 * 3, timestamp=None):
"""Push a message onto the queue"""
cl = self.cl or self._get_cl(consistency)
if not timestamp:
now = uuid.uuid1()
elif isinstance(timestamp, (float, Decimal)):
now = convert_time_to_uuid(timestamp, randomize=True)
else:
now = uuid.UUID(hex=timestamp)
queue_name = '%s:%s' % (application_name, queue_name)
if metadata:
batch = pycassa.batch.Mutator(self.pool,
write_consistency_level=cl)
batch.insert(self.message_fam, key=queue_name,
columns={now: message}, ttl=ttl)
batch.insert(self.meta_fam, key=now, columns=metadata, ttl=ttl)
batch.send()
else:
self.message_fam.insert(key=queue_name, columns={now: message},
ttl=ttl, write_consistency_level=cl)
timestamp = Decimal(now.time - 0x01b21dd213814000L) / DECIMAL_1E7
return now.hex, timestamp
def retrieve(self, consistency, application_name, queue_name, message_id,
include_metadata=False):
"""Retrieve a single message"""
if isinstance(message_id, basestring):
# Convert to uuid for lookup
message_id = uuid.UUID(hex=message_id)
else:
# Assume its a float/decimal, convert to UUID
message_id = convert_time_to_uuid(message_id)
queue_name = '%s:%s' % (application_name, queue_name)
queue = message_store[queue_name]
found = None
for msg in queue:
if msg.id == message_id:
found = msg
break
if not found:
return {}
now = Decimal(repr(time.time()))
if found.expiration and now > found.expiration:
queue.remove(found)
return {}
obj = {
'message_id': found.id.hex,
'timestamp': (Decimal(found.id.time - 0x01b21dd213814000L) /
DECIMAL_1E7),
'body': found.body,
'metadata': {},
'queue_name': queue_name[queue_name.find(':'):]
}
if include_metadata:
obj['metadata'] = found.metadata
return obj
def push(self,
consistency,
application_name,
queue_name,
message,
metadata=None,
ttl=60 * 60 * 24 * 3,
timestamp=None):
"""Push a message onto the queue"""
if not timestamp:
now = uuid.uuid1()
elif isinstance(timestamp, (float, Decimal)):
now = convert_time_to_uuid(timestamp, randomize=True)
else:
now = uuid.UUID(hex=timestamp)
msg = Message(id=now, body=message, ttl=ttl)
if metadata:
msg.metadata = metadata
timestamp = Decimal(msg.id.time - 0x01b21dd213814000L) / DECIMAL_1E7
queue_name = '%s:%s' % (application_name, queue_name)
if msg in message_store[queue_name]:
message_store[queue_name].remove(msg)
message_store[queue_name].append(msg)
return msg.id.hex, timestamp
def retrieve_batch(self, consistency, application_name, queue_names,
limit=None, include_metadata=False, start_at=None,
order="ascending"):
"""Retrieve a batch of messages off the queue"""
if not isinstance(queue_names, list):
raise Exception("queue_names must be a list")
order = -1 if order == 'descending' else 1
if start_at:
if isinstance(start_at, basestring):
# Assume its a hex, transform to a UUID
start_at = uuid.UUID(hex=start_at)
else:
# Assume its a float/decimal, convert to UUID
start_at = convert_time_to_uuid(start_at)
queue_names = ['%s:%s' % (application_name, x) for x in queue_names]
results = []
now = Decimal(repr(time.time()))
for queue_name in queue_names:
msgs = message_store[queue_name]
if not msgs:
continue
msgs.sort(key=lambda k: (k.id.time, k.id.bytes))
if start_at:
# Locate the index given the start_at
point = (start_at.time, start_at.bytes)
beg = msgs[0].id
end = msgs[-1].id
if point <= (beg.time, beg.bytes):
# Is the start_at less than the beginning? Start at beginning
start = 0
elif point >= (end.time, end.bytes):
# Is the start_at larger than the end? Start at the end
start = len(msgs) - 1
else:
# The start point is somewhere inside, skim through until
# we hit a value a value equal to or greater than
start = 0
msg_comp = (msgs[start].id.time, msgs[start].id.bytes)
while point > msg_comp:
start += 1
msg_comp = (msgs[start].id.time, msgs[start].id.bytes)
else:
if order == -1:
start = len(msgs) - 1
else:
start = 0
count = 0
for msg in msgs[start::order]:
if msg.expiration and now > msg.expiration:
msgs.remove(msg)
continue
count += 1
if limit and count > limit:
break
obj = {
'message_id': msg.id.hex,
'timestamp': (Decimal(msg.id.time - 0x01b21dd213814000L) /
DECIMAL_1E7),
'body': msg.body,
'metadata': {},
'queue_name': queue_name[queue_name.find(':'):]
}
if include_metadata:
obj['metadata'] = msg.metadata
results.append(obj)
return results
def retrieve_batch(self,
consistency,
application_name,
queue_names,
limit=None,
include_metadata=False,
start_at=None,
order="ascending"):
"""Retrieve a batch of messages off the queue"""
if not isinstance(queue_names, list):
raise Exception("queue_names must be a list")
cl = self.cl or self._get_cl(consistency)
delay = self._get_delay(consistency)
kwargs = {'read_consistency_level': cl}
if order == 'descending':
kwargs['column_reversed'] = True
if limit:
kwargs['column_count'] = limit
if start_at:
if isinstance(start_at, basestring):
# Assume its a hex, transform to a datetime
start_at = uuid.UUID(hex=start_at)
else:
# Assume its a float/decimal, convert to UUID
start_at = convert_time_to_uuid(start_at)
kwargs['column_start'] = start_at
queue_names = ['%s:%s' % (application_name, x) for x in queue_names]
results = self.message_fam.multiget(keys=queue_names, **kwargs)
results = results.items()
if delay:
cut_off = time.time() - delay
# Turn it into time in ns, for efficient comparison
cut_off = int(cut_off * 1e7) + 0x01b21dd213814000L
result_list = []
msg_hash = {}
for queue_name, messages in results:
for msg_id, body in messages.items():
if delay and msg_id.time >= cut_off:
continue
obj = {
'message_id':
msg_id.hex,
'timestamp':
(Decimal(msg_id.time - 0x01b21dd213814000L) / DECIMAL_1E7),
'body':
body,
'metadata': {},
'queue_name':
queue_name[queue_name.find(':'):]
}
result_list.append(obj)
msg_hash[msg_id] = obj
# Get metadata?
if include_metadata:
results = self.meta_fam.multiget(keys=msg_hash.keys())
for msg_id, metadata in results.items():
msg_hash[msg_id]['metadata'] = metadata
return result_list
def retrieve_batch(self,
consistency,
application_name,
queue_names,
limit=None,
include_metadata=False,
start_at=None,
order="ascending"):
"""Retrieve a batch of messages off the queue"""
if not isinstance(queue_names, list):
raise Exception("queue_names must be a list")
order = -1 if order == 'descending' else 1
if start_at:
if isinstance(start_at, basestring):
# Assume its a hex, transform to a UUID
start_at = uuid.UUID(hex=start_at)
else:
# Assume its a float/decimal, convert to UUID
start_at = convert_time_to_uuid(start_at)
queue_names = ['%s:%s' % (application_name, x) for x in queue_names]
results = []
now = Decimal(repr(time.time()))
for queue_name in queue_names:
msgs = message_store[queue_name]
if not msgs:
continue
msgs.sort(key=lambda k: (k.id.time, k.id.bytes))
if start_at:
# Locate the index given the start_at
point = (start_at.time, start_at.bytes)
beg = msgs[0].id
end = msgs[-1].id
if point <= (beg.time, beg.bytes):
# Is the start_at less than the beginning? Start at beginning
start = 0
elif point >= (end.time, end.bytes):
# Is the start_at larger than the end? Start at the end
start = len(msgs) - 1
else:
# The start point is somewhere inside, skim through until
# we hit a value a value equal to or greater than
start = 0
msg_comp = (msgs[start].id.time, msgs[start].id.bytes)
while point > msg_comp:
start += 1
msg_comp = (msgs[start].id.time, msgs[start].id.bytes)
else:
if order == -1:
start = len(msgs) - 1
else:
start = 0
count = 0
for msg in msgs[start::order]:
if msg.expiration and now > msg.expiration:
msgs.remove(msg)
continue
count += 1
if limit and count > limit:
break
obj = {
'message_id':
msg.id.hex,
'timestamp':
(Decimal(msg.id.time - 0x01b21dd213814000L) / DECIMAL_1E7),
'body':
msg.body,
'metadata': {},
'queue_name':
queue_name[queue_name.find(':'):]
}
if include_metadata:
obj['metadata'] = msg.metadata
results.append(obj)
return results
