class HyperdexIndex():
"""
tag list format:
"user": { "tags": set of tags,
"created": date int,
"modified": date int
"size": int
}
message format:
"uuid": { "tags": set[])
"flags": set([])
"to": "to header",
"from": "from header",
"cc": "cc header",
"bcc": "bcc header",
"subject": subject header",
"size": size int,
"date": "date header",
"dateint": date int
"storeddate": unix timestamp int,
}
The index internally stores this as is, with the addition of a
'date_int' field containing unixtimestamp version of the Date header
for fast indexing, as well as json encoded versions of the tags
and space delimited version of the flags.
"""
def __init__(self, config, user, readonly):
self.client = Client(config['host'], int(config['port']))
self.user = str(user)
self.tags = config['tag_space']
self.messages = config['message_space']
def _init_user(self):
if not self.client.get(self.tags, self.user):
self.client.put(self.tags, self.user, {'tags': set([]), 'created': 0})
def _parse_date(self, date):
if date is None:
dateint = 0
else:
dtime = utils.parsedate_tz(date)
if dtime is None:
try:
dtime = parse(date)
except:
dtime = datetime.now()
dtime = dtime.timetuple()+ (0,)
dateint = int(utils.mktime_tz(dtime))
return dateint
def _update_size(self, delta):
self.client.atomic_add(self.tags, self.user, {'size': delta})
def _update_modified(self):
self.client.put(self.tags, self.user, {'modified': int(time.time())})
def _update_tag_count(self, tag, delta):
self.client.map_atomic_add(self.tags, self.user, {'tag_count': {str(tag): delta}})
def _update_tag_unread_count(self, tag, delta):
self.client.map_atomic_add(self.tags, self.user, {'tag_unread_count': {str(tag): delta}})
def _update_tag_modified(self, tag):
self.client.map_add(self.tags, self.user, {'tag_modified': {str(tag): int(time.time())}})
def _number_compare(self, verb, number):
if verb is filter.FilterVerb.Contains:
return Equals(number)
if verb is filter.FilterVerb.Less:
return LessThan(number)
if verb is filter.FilterVerb.Greater:
return GreaterThan(number)
def list_messages(self, filterlist=[], sort=None, limit=None):
filters = {'user': self.user}
for filter, verb, value in filterlist:
if filter in ['to', 'from', 'cc',
'bcc', 'subject']:
filters[filter] = Regex(str(value))
elif filter in ['stored', 'size']:
filters[filter] = self._number_compare(verb, int(value))
elif filter in ['date']:
d = time.mktime(utils.parsedate(value))
filters['dateint'] = self._number_compare(verb, int(d))
elif filter in ['tag']:
filters['tags'] = Contains(str(value))
elif filter in ['flag']:
filters['flags'] = Contains(str(value))
if sort is None or sort[0] is None:
sortfield = 'uuid'
elif sort[0] == 'date':
sortfield = 'dateint'
else:
sortfield = str(sort[0])
if not sort or sort[1] is True:
sortdir = 'maximize'
else:
sortdir = 'minimize'
if limit is None:
limit = (5000000, 0)
print filters
print sort
print sortdir
print limit
res = [msg['uuid'] for msg in self.client.sorted_search(self.messages, filters, sortfield, limit[0] + limit[1], sortdir)]
return res[0:len(res) - limit[1]][::-1]
def list_tags(self):
self._init_user()
tags = self.client.get(self.tags, self.user)['tags']
return list(tags) or []
def get_tag(self, tag):
if not tag in self.list_tags():
return None
tagdata = self.client.get(self.tags, self.user)
r = {"count": tagdata['tag_count'][tag],
"unread": tagdata['tag_unread_count'][tag],
"created": tagdata['tag_created'][tag],
"last-modified": tagdata['tag_modified'][tag]}
return r
def put_tag(self, tag):
if not self.client.get(self.tags, self.user):
self._init_user()
self.client.set_add(self.tags, self.user, {'tags': str(tag)})
self.client.map_add(self.tags, self.user, {'tag_unread_count': {str(tag): 0}})
self.client.map_add(self.tags, self.user, {'tag_count': {str(tag): 0}})
self.client.map_add(self.tags, self.user, {'tag_created': {str(tag): int(time.time())}})
self.client.map_add(self.tags, self.user, {'tag_modified': {str(tag): int(time.time())}})
def del_tag(self, tag):
pass
def get_message(self, uuid):
msg = self.client.get(self.messages, uuid)
del msg['dateint']
del msg['user']
msg['tags'] = list(msg['tags'])
msg['flags'] = list(msg['flags'])
return msg
def put_message_tags(self, uuid, newtags):
uuid = str(uuid)
totaltags = self.client.get(self.tags, self.user)['tags']
oldtags = self.client.get(self.messages, uuid)['tags']
flags = self.client.get(self.messages, uuid)['flags']
if not set(newtags).issubset(totaltags):
raise TagNotFound()
for old in oldtags:
if old not in newtags:
self._update_tag_modified(old)
self._update_tag_count(old, -1)
if "\\Seen" not in flags:
self._update_tag_unread_count(old, -1)
for new in newtags:
if new not in oldtags:
self._update_tag_count(new, 1)
self._update_tag_modified(new)
if "\\Seen" not in flags:
self._update_tag_unread_count(new, 1)
self.client.put(self.messages, str(uuid), {'tags': set(newtags)})
def put_message_flags(self, uuid, flags):
uuid = str(uuid)
oldflags = self.client.get(self.messages, uuid)['flags']
tags = self.client.get(self.tags, self.user)['tags']
self.client.put(self.messages, str(uuid), {'flags': set([str(x) for x in flags])})
if "\\Seen" in oldflags and "\\Seen" not in flags:
for tag in tags:
self._update_tag_unread_count(tag, 1)
if "\\Seen" not in oldflags and "\\Seen" in flags:
for tag in tags:
self._update_tag_unread_count(tag, -1)
for tag in tags:
self._update_tag_modified(tag)
def _encode_strings(self, msg):
for field in ['to', 'from', 'cc', 'bcc', 'subject', 'date']:
msg[field] = msg[field].decode('iso-8859-1').encode('utf-8')
msg['flags'] = [x.decode('iso-8859-1').encode('utf-8') for x in msg['flags']]
msg['tags'] = [x.decode('iso-8859-1').encode('utf-8') for x in msg['tags']]
return msg
def put_message(self, uuid, msg):
self._init_user()
msg = self._encode_strings(msg)
msg['tags'] = set(msg['tags'])
msg['flags'] = set(msg['flags'])
msg['dateint'] = self._parse_date(msg['date'])
msg['user'] = self.user
self.client.put(self.messages, uuid, msg)
self._update_size(msg['size'])
for tag in msg['tags']:
self._update_tag_modified(tag)
self._update_tag_count(tag, 1)
if '\\Seen' not in msg['flags']:
self._update_tag_unread_count(tag, 1)
self._update_modified()
def del_message(self, uuid):
msg = self.get_message(uuid)
self.client.delete(self.messages, uuid)
self._update_size(-msg['size'])
for tag in msg['tags']:
self._update_tag_modified(tag)
self._update_tag_count(tag, -1)
if '\\Seen' in msg['flags']:
self._update_tag_unread_count(tag, -1)
self._update_modified()
def get_user(self):
self._init_user()
row = self.client.get(self.tags, self.user)
count = sum(row['tag_count'].values())
u = {"user": self.user,
"created": row['created'],
"modified": row['modified'],
"size": row['size'],
"count": count
}
return u
