def __init__(self, address, port, graph_name):
'''
Constructor.
Args:
address: The IP address of HyperDex's coordinator process (str).
port: The port number of HyperDex's coordinator process (int).
Returns:
An instance of the HyperDexStore, which wraps the underlying connection.
'''
self._address = address
self._port = port
self._graph_name = graph_name
self.hyperdex_client = Client(address,
port) #HyperDex python binding (Client)
self.hyperdex_admin = Admin(address,
port) #HyperDex python binding (Admin)
self.sysadmin = SysAdmin(address, port, graph_name,
self.hyperdex_client, self.hyperdex_admin)
self.typeadmin = TypeAdmin(address, port, graph_name,
self.hyperdex_client, self.hyperdex_admin)
#validate the underlying database for the particular components, i.e.
#checking for the system/description space
self.sysadmin.validate_database()
self.typeadmin.validate_database()
self._generic_vertex = graph_name + '_generic_vertex'
self.validate_database()
class HyperDexStore():
'''
The HyperDexStore class provides the first level of abstraction towards the client. It
wraps the HyperDex python language binding. It is used by the HyperDexGraph API and
handles the underlying connection to the database.
'''
def __init__(self, address, port, graph_name):
'''
Constructor.
Args:
address: The IP address of HyperDex's coordinator process (str).
port: The port number of HyperDex's coordinator process (int).
Returns:
An instance of the HyperDexStore, which wraps the underlying connection.
'''
self._address = address
self._port = port
self._graph_name = graph_name
self.hyperdex_client = Client(address,
port) #HyperDex python binding (Client)
self.hyperdex_admin = Admin(address,
port) #HyperDex python binding (Admin)
self.sysadmin = SysAdmin(address, port, graph_name,
self.hyperdex_client, self.hyperdex_admin)
self.typeadmin = TypeAdmin(address, port, graph_name,
self.hyperdex_client, self.hyperdex_admin)
#validate the underlying database for the particular components, i.e.
#checking for the system/description space
self.sysadmin.validate_database()
self.typeadmin.validate_database()
self._generic_vertex = graph_name + '_generic_vertex'
self.validate_database()
def validate_database(self):
try:
self.hyperdex_client.get(self._generic_vertex, 1)
logging.info('Generic vertices available')
except HyperClientException, e:
if e.symbol() == 'HYPERDEX_CLIENT_UNKNOWNSPACE':
logging.info('Creating space generic_vertex')
self.hyperdex_admin.add_space('''
space ''' + self._generic_vertex + '''
key int id
attributes
map(int, string) incoming_edges,
map(int, string) outgoing_edges,
string value
''')
os.system('hyperdex wait-until-stable -h {} -p {}'.format(
self._address, self._port))
return
logging.error(str(e))
class HyperDexStore():
'''
The HyperDexStore class provides the first level of abstraction towards the client. It
wraps the HyperDex python language binding. It is used by the HyperDexGraph API and
handles the underlying connection to the database.
'''
def __init__(self, address, port, graph_name):
'''
Constructor.
Args:
address: The IP address of HyperDex's coordinator process (str).
port: The port number of HyperDex's coordinator process (int).
Returns:
An instance of the HyperDexStore, which wraps the underlying connection.
'''
self._address = address
self._port = port
self._graph_name = graph_name
self.hyperdex_client = Client(address, port)#HyperDex python binding (Client)
self.hyperdex_admin = Admin(address, port)#HyperDex python binding (Admin)
self.sysadmin = SysAdmin(address, port, graph_name, self.hyperdex_client, self.hyperdex_admin)
self.typeadmin = TypeAdmin(address, port, graph_name, self.hyperdex_client, self.hyperdex_admin)
#validate the underlying database for the particular components, i.e.
#checking for the system/description space
self.sysadmin.validate_database()
self.typeadmin.validate_database()
self._generic_vertex = graph_name + '_generic_vertex'
self.validate_database()
def validate_database(self):
try:
self.hyperdex_client.get(self._generic_vertex, 1)
logging.info('Generic vertices available')
except HyperClientException, e:
if e.symbol() == 'HYPERDEX_CLIENT_UNKNOWNSPACE':
logging.info('Creating space generic_vertex')
self.hyperdex_admin.add_space('''
space '''+ self._generic_vertex + '''
key int id
attributes
map(int, string) incoming_edges,
map(int, string) outgoing_edges,
string value
''')
os.system('hyperdex wait-until-stable -h {} -p {}'.format(self._address, self._port))
return
logging.error(str(e))
def __init__(self, address, port, graph_name):
'''
Constructor.
Args:
address: The IP address of HyperDex's coordinator process (str).
port: The port number of HyperDex's coordinator process (int).
Returns:
An instance of the HyperDexStore, which wraps the underlying connection.
'''
self._address = address
self._port = port
self._graph_name = graph_name
self.hyperdex_client = Client(address, port)#HyperDex python binding (Client)
self.hyperdex_admin = Admin(address, port)#HyperDex python binding (Admin)
self.sysadmin = SysAdmin(address, port, graph_name, self.hyperdex_client, self.hyperdex_admin)
self.typeadmin = TypeAdmin(address, port, graph_name, self.hyperdex_client, self.hyperdex_admin)
#validate the underlying database for the particular components, i.e.
#checking for the system/description space
self.sysadmin.validate_database()
self.typeadmin.validate_database()
self._generic_vertex = graph_name + '_generic_vertex'
self.validate_database()
class FHIRRPCAccessLayer(xmlrpc.XMLRPC):
def connect_to_hyperdex(self, url=RPC_URL, port=RPC_PORT):
self.client = Client(RPC_URL, RPC_PORT)
print("Connected to {0}:{1}".format(RPC_URL,RPC_PORT))
def xmlrpc_echo(self, x):
return x
def xmlrpc_read(self, fhir_type, fhir_id):
R = [x for x in self.client.search('resource_version',{'fhir_id':fhir_id})][0]
m = VERSION_ID_PATTERN.match(R['content'])
# Correct the version_id weirdness
if m:
R['version_id'] = int(m.group(1))
for r in R:
print(r,R[r])
return R
class FHIRRPCAccessLayer(xmlrpc.XMLRPC):
def connect_to_hyperdex(self, url=RPC_URL, port=RPC_PORT):
self.client = Client(RPC_URL, RPC_PORT)
print("Connected to {0}:{1}".format(RPC_URL, RPC_PORT))
def xmlrpc_echo(self, x):
return x
def xmlrpc_read(self, fhir_type, fhir_id):
R = [
x for x in self.client.search('resource_version',
{'fhir_id': fhir_id})
][0]
m = VERSION_ID_PATTERN.match(R['content'])
# Correct the version_id weirdness
if m:
R['version_id'] = int(m.group(1))
for r in R:
print(r, R[r])
return R
def connect_to_hyperdex(self, url=RPC_URL, port=RPC_PORT):
self.client = Client(RPC_URL, RPC_PORT)
print("Connected to {0}:{1}".format(RPC_URL,RPC_PORT))
def setUp(self):
self.hyperdex = Admin(ADDRESS, PORT)
self.hyperdex_client = Client(ADDRESS, PORT)
self.g = HyperDexGraph(ADDRESS, PORT, GRAPH)
def connect_to_hyperdex(self, url=RPC_URL, port=RPC_PORT):
self.client = Client(RPC_URL, RPC_PORT)
print("Connected to {0}:{1}".format(RPC_URL, RPC_PORT))
def __init__(self, config, user, readonly):
self.client = Client(config['host'], int(config['port']))
self.user = str(user)
self.tags = config['tag_space']
self.messages = config['message_space']
class HyperdexIndex():
"""
tag list format:
"user": { "tags": set of tags,
"created": date int,
"modified": date int
"size": int
}
message format:
"uuid": { "tags": set[])
"flags": set([])
"to": "to header",
"from": "from header",
"cc": "cc header",
"bcc": "bcc header",
"subject": subject header",
"size": size int,
"date": "date header",
"dateint": date int
"storeddate": unix timestamp int,
}
The index internally stores this as is, with the addition of a
'date_int' field containing unixtimestamp version of the Date header
for fast indexing, as well as json encoded versions of the tags
and space delimited version of the flags.
"""
def __init__(self, config, user, readonly):
self.client = Client(config['host'], int(config['port']))
self.user = str(user)
self.tags = config['tag_space']
self.messages = config['message_space']
def _init_user(self):
if not self.client.get(self.tags, self.user):
self.client.put(self.tags, self.user, {'tags': set([]), 'created': 0})
def _parse_date(self, date):
if date is None:
dateint = 0
else:
dtime = utils.parsedate_tz(date)
if dtime is None:
try:
dtime = parse(date)
except:
dtime = datetime.now()
dtime = dtime.timetuple()+ (0,)
dateint = int(utils.mktime_tz(dtime))
return dateint
def _update_size(self, delta):
self.client.atomic_add(self.tags, self.user, {'size': delta})
def _update_modified(self):
self.client.put(self.tags, self.user, {'modified': int(time.time())})
def _update_tag_count(self, tag, delta):
self.client.map_atomic_add(self.tags, self.user, {'tag_count': {str(tag): delta}})
def _update_tag_unread_count(self, tag, delta):
self.client.map_atomic_add(self.tags, self.user, {'tag_unread_count': {str(tag): delta}})
def _update_tag_modified(self, tag):
self.client.map_add(self.tags, self.user, {'tag_modified': {str(tag): int(time.time())}})
def _number_compare(self, verb, number):
if verb is filter.FilterVerb.Contains:
return Equals(number)
if verb is filter.FilterVerb.Less:
return LessThan(number)
if verb is filter.FilterVerb.Greater:
return GreaterThan(number)
def list_messages(self, filterlist=[], sort=None, limit=None):
filters = {'user': self.user}
for filter, verb, value in filterlist:
if filter in ['to', 'from', 'cc',
'bcc', 'subject']:
filters[filter] = Regex(str(value))
elif filter in ['stored', 'size']:
filters[filter] = self._number_compare(verb, int(value))
elif filter in ['date']:
d = time.mktime(utils.parsedate(value))
filters['dateint'] = self._number_compare(verb, int(d))
elif filter in ['tag']:
filters['tags'] = Contains(str(value))
elif filter in ['flag']:
filters['flags'] = Contains(str(value))
if sort is None or sort[0] is None:
sortfield = 'uuid'
elif sort[0] == 'date':
sortfield = 'dateint'
else:
sortfield = str(sort[0])
if not sort or sort[1] is True:
sortdir = 'maximize'
else:
sortdir = 'minimize'
if limit is None:
limit = (5000000, 0)
print filters
print sort
print sortdir
print limit
res = [msg['uuid'] for msg in self.client.sorted_search(self.messages, filters, sortfield, limit[0] + limit[1], sortdir)]
return res[0:len(res) - limit[1]][::-1]
def list_tags(self):
self._init_user()
tags = self.client.get(self.tags, self.user)['tags']
return list(tags) or []
def get_tag(self, tag):
if not tag in self.list_tags():
return None
tagdata = self.client.get(self.tags, self.user)
r = {"count": tagdata['tag_count'][tag],
"unread": tagdata['tag_unread_count'][tag],
"created": tagdata['tag_created'][tag],
"last-modified": tagdata['tag_modified'][tag]}
return r
def put_tag(self, tag):
if not self.client.get(self.tags, self.user):
self._init_user()
self.client.set_add(self.tags, self.user, {'tags': str(tag)})
self.client.map_add(self.tags, self.user, {'tag_unread_count': {str(tag): 0}})
self.client.map_add(self.tags, self.user, {'tag_count': {str(tag): 0}})
self.client.map_add(self.tags, self.user, {'tag_created': {str(tag): int(time.time())}})
self.client.map_add(self.tags, self.user, {'tag_modified': {str(tag): int(time.time())}})
def del_tag(self, tag):
pass
def get_message(self, uuid):
msg = self.client.get(self.messages, uuid)
del msg['dateint']
del msg['user']
msg['tags'] = list(msg['tags'])
msg['flags'] = list(msg['flags'])
return msg
def put_message_tags(self, uuid, newtags):
uuid = str(uuid)
totaltags = self.client.get(self.tags, self.user)['tags']
oldtags = self.client.get(self.messages, uuid)['tags']
flags = self.client.get(self.messages, uuid)['flags']
if not set(newtags).issubset(totaltags):
raise TagNotFound()
for old in oldtags:
if old not in newtags:
self._update_tag_modified(old)
self._update_tag_count(old, -1)
if "\\Seen" not in flags:
self._update_tag_unread_count(old, -1)
for new in newtags:
if new not in oldtags:
self._update_tag_count(new, 1)
self._update_tag_modified(new)
if "\\Seen" not in flags:
self._update_tag_unread_count(new, 1)
self.client.put(self.messages, str(uuid), {'tags': set(newtags)})
def put_message_flags(self, uuid, flags):
uuid = str(uuid)
oldflags = self.client.get(self.messages, uuid)['flags']
tags = self.client.get(self.tags, self.user)['tags']
self.client.put(self.messages, str(uuid), {'flags': set([str(x) for x in flags])})
if "\\Seen" in oldflags and "\\Seen" not in flags:
for tag in tags:
self._update_tag_unread_count(tag, 1)
if "\\Seen" not in oldflags and "\\Seen" in flags:
for tag in tags:
self._update_tag_unread_count(tag, -1)
for tag in tags:
self._update_tag_modified(tag)
def _encode_strings(self, msg):
for field in ['to', 'from', 'cc', 'bcc', 'subject', 'date']:
msg[field] = msg[field].decode('iso-8859-1').encode('utf-8')
msg['flags'] = [x.decode('iso-8859-1').encode('utf-8') for x in msg['flags']]
msg['tags'] = [x.decode('iso-8859-1').encode('utf-8') for x in msg['tags']]
return msg
def put_message(self, uuid, msg):
self._init_user()
msg = self._encode_strings(msg)
msg['tags'] = set(msg['tags'])
msg['flags'] = set(msg['flags'])
msg['dateint'] = self._parse_date(msg['date'])
msg['user'] = self.user
self.client.put(self.messages, uuid, msg)
self._update_size(msg['size'])
for tag in msg['tags']:
self._update_tag_modified(tag)
self._update_tag_count(tag, 1)
if '\\Seen' not in msg['flags']:
self._update_tag_unread_count(tag, 1)
self._update_modified()
def del_message(self, uuid):
msg = self.get_message(uuid)
self.client.delete(self.messages, uuid)
self._update_size(-msg['size'])
for tag in msg['tags']:
self._update_tag_modified(tag)
self._update_tag_count(tag, -1)
if '\\Seen' in msg['flags']:
self._update_tag_unread_count(tag, -1)
self._update_modified()
def get_user(self):
self._init_user()
row = self.client.get(self.tags, self.user)
count = sum(row['tag_count'].values())
u = {"user": self.user,
"created": row['created'],
"modified": row['modified'],
"size": row['size'],
"count": count
}
return u
def setUp(self):
self.hyperdex_client = Client(ADDRESS, PORT)
self.hyperdex_admin = Admin(ADDRESS, PORT)
self.typeadmin = TypeAdmin(ADDRESS, PORT, GRAPH, self.hyperdex_client,
self.hyperdex_admin)
self.typeadmin.validate_database()
