def __init__(self, transport, address, p_id, db_n):
self.engine = Engine(transport)
self.server_addr = address
self.provider_id = p_id
self.db_name = db_n
# init the engine
self.client = SDSKVClient(self.engine)
self.addr = self.engine.lookup(self.server_addr)
self.provider_handle = self.client.create_provider_handle(self.addr, self.provider_id)
self.db = self.provider_handle.open(self.db_name)
# create a dictionary for values
self.command_dic = {}
self.used_unique_ids = []
self.count = 0
self.addingKeys = False
# Generate Hash
ip_addr = self.__getIP()
self.hash_val = self.__generateHash(ip_addr)
# excluded keys
self.removed_keys = []
def __init__(self, pdb_path='', pdb_sharded_num=0, pdb_addr=''):
if pdb_addr == '': # Standalone mode, need to create engine provider
self.pdb_engine = Engine('ofi+tcp', mode=pymargo.server)
self.pdb_provider = SonataProvider(self.pdb_engine, 0)
self.pdb_address = str(self.pdb_engine.addr())
self.pdb_admin = SonataAdmin(self.pdb_engine)
self.pdb_client = SonataClient(self.pdb_engine)
if pdb_path and int(pdb_sharded_num) > 0:
for i in range(int(pdb_sharded_num)):
pdb_name = 'provdb.' + str(i)
file_name = pdb_path + pdb_name + '.unqlite'
self.pdb_admin.attach_database(
self.pdb_address, 0, pdb_name, 'unqlite',
"{ \"path\" : \"%s\" }" % file_name)
else: # Other Chimbuko module created the engine
self.pdb_engine = Engine(pdb_addr.split(':')[0], pymargo.client)
self.pdb_address = pdb_addr
self.pdb_client = SonataClient(self.pdb_engine)
self.pdb_databases = []
self.pdb_collections = []
self.pdb_names = []
for i in range(pdb_sharded_num):
pdb_name = 'provdb.' + str(i)
self.pdb_names.append(pdb_name)
database = self.pdb_client.open(self.pdb_address, 0, pdb_name)
self.pdb_databases.append(database)
col = database.open('anomalies')
self.pdb_collections.append(col)
def setUpClass(cls):
cls._engine = Engine('tcp://127.0.0.1:1234')
cls._provider = SDSKVProvider(cls._engine, 1)
cls._path = tempfile.mkdtemp()
cls._db_id = cls._provider.attach_database("mydatabase", cls._path,
pysdskv.server.leveldb)
cls._client = SDSKVClient(cls._engine)
cls._addr = cls._engine.lookup('tcp://127.0.0.1:1234')
cls._ph = cls._client.create_provider_handle(cls._addr, 1)
def test(address, nshard):
with Engine(address.split(':')[0], pymargo.client) as engine:
client = SonataClient(engine)
for i in range(nshard):
pdb_name = 'provdb.' + str(i)
pdb = client.open(address, 0, pdb_name)
col = pdb.open('anomalies')
print("Shard {} has the size of {}.".format(i, col.size))
del pdb
del col
del address
del client
def setUpClass(cls):
cls._engine = Engine('tcp://127.0.0.1:1234')
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
import sys
sys.path.append('build/lib.linux-x86_64-3.7')
from pymargo.core import Engine
from pysdskv.client import *
engine = Engine('ofi+tcp')
server_addr = sys.argv[1]
provider_id = int(sys.argv[2])
client = SDSKVClient(engine)
addr = engine.lookup(server_addr)
provider_handle = client.create_provider_handle(addr, provider_id)
db = provider_handle.open("mydatabase")
db.put("matthieu", "*****@*****.**")
val = db.get("matthieu")
print("db.get('matthieu') returned " + str(val))
try:
val = db.get("phil")
print("db.get('phil') returned " + str(val))
except KeyError:
print("Correctly throws an exception when trying to lookup 'phil'")
e = db.exists("matthieu")
print("db.exists('matthieu') returned " + str(e))
e = db.exists("phil")
print("db.exists('phil') returned " + str(e))
db.erase("matthieu")
try:
localBulk = engine.create_bulk(localArray, pymargo.bulk.read_write)
remoteBulk = Bulk.from_base64(engine, bulk_str)
print("Remote bulk deserialized")
size = 5 * 7 * localArray.itemsize
engine.transfer(pymargo.bulk.pull, handle.get_addr(), remoteBulk,
0, localBulk, 0, size)
except Exception as error:
print("An exception was caught:")
print(error)
print("Transfer done")
print("Received: ")
print(localArray)
handle.respond("OK")
engine.finalize()
def WhenFinalize():
print("Finalize was called")
engine = Engine('tcp')
provider_id = 42
print("Server running at address " + str(engine.addr()) +
" with provider_id " + str(provider_id))
engine.on_finalize(WhenFinalize)
provider = HelloProvider(engine, provider_id)
engine.wait_for_finalize()
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
import sys
sys.path.append('build/lib.linux-x86_64-3.7')
from pymargo.core import Engine
import pysdskv.server
from pysdskv.server import SDSKVProvider
engine = Engine('ofi+tcp')
engine.enable_remote_shutdown()
provider_id = 42
print("Server running at address " + str(engine.addr()) +
" with provider_id=" + str(provider_id))
provider = SDSKVProvider(engine, provider_id)
dbid = provider.attach_database("mydatabase", "/tmp/sdskv",
pysdskv.server.leveldb)
print("Created a database with id " + str(dbid))
engine.wait_for_finalize()
print(record_ids)
# fetch multiple records
fetched_records = [
json.loads(x) for x in collection.fetch_multi(record_ids)
]
print(fetched_records)
# get all the records
all_records = [json.loads(x) for x in collection.all]
print(all_records)
# get the size of the collection
print(collection.size)
# get the last record id in the collection
print(collection.last_record_id)
# update record 1 (Lucas)
collection.update(1, json.dumps({"name": "Lucas", "age": 30}))
print(json.loads(collection.fetch(1)))
# find out all the records where age > 30
jx9_filter = "function($user) { return $user.age > 30; }"
filtered_records = [json.loads(x) for x in collection.filter(jx9_filter)]
print(filtered_records)
# destroy database
admin.destroy_database(address, 0, 'mydb')
if __name__ == '__main__':
with Engine('na+sm', pymargo.server) as engine:
provider = SonataProvider(engine, 0)
test(engine)
del provider
engine.finalize()
class InWrapMochi:
def __init__(self, transport, address, p_id, db_n):
self.engine = Engine(transport)
self.server_addr = address
self.provider_id = p_id
self.db_name = db_n
# init the engine
self.client = SDSKVClient(self.engine)
self.addr = self.engine.lookup(self.server_addr)
self.provider_handle = self.client.create_provider_handle(self.addr, self.provider_id)
self.db = self.provider_handle.open(self.db_name)
# create a dictionary for values
self.command_dic = {}
self.used_unique_ids = []
self.count = 0
self.addingKeys = False
def put_keyval(self, key, val):
self.db.put(key, val)
def del_key(self, key):
if (self.db.exists(key)):
self.db.erase(key)
else:
return "key does not exist"
def get_val(self, key):
if (self.db.exists(key)):
return self.db.get(key)
else:
return "key does not exist"
def exists(self, key):
return self.db.exists(key)
def list_keyVal(self):
key,val = self.db.list_keyvals()
return key,val
def shutdown(self):
self.client.shutdown_service(self.addr)
def getIP(self):
hostname = socket.gethostname()
IPAddr = socket.gethostbyname(hostname)
return str(IPAddr)
def generateHash(self, value):
hash_object = hashlib.md5(value.encode())
temp_str = str(hash_object.hexdigest())
return ( temp_str[0:8] )
# Transaction operations: Set and commit
def set(self, key, value):
self.command_dic[key + "-" + str(self.count)] = value
self.count = self.count + 1
def commit(self):
ip_addr = self.getIP()
hash_val = self.generateHash(ip_addr)
new_key = "NEW_KEY_" + hash_val
# Loop until you can add a new key
while ( self.db.exists(new_key) ):
continue
# Generate a random number to pad keys with
#rand_num = random.randint(99999,1000000)
#while rand_num in self.used_unique_ids:
# rand_num = random.randint(99999,1000000)
#self.used_unique_ids.append(rand_num)
# Format of keys:
# XXXXXX:Key-YYY
# Add keys to mochi DB
key_list = []
self.addingKeys = True
#self.db.put("NEW_KEY", "0") # adding keys
for key,val in self.command_dic.items():
key_padded = hash_val + ":" + key # add unique tag to each
#key_padded = key + ":" + str(rand_num) # add unique tag to each
self.db.put(key_padded, val) # put key in mochi db
key_list.append(key) # save the key for later deletion
print("key: ", key_padded)
print("val: ", val)
#self.db.put("NEW_KEY", str(rand_num)) # adding keys done
self.db.put(new_key, hash_val) # adding keys done
self.addingKeys = False
# Delete keys
for k in key_list:
del self.command_dic[k]
import sys
import numpy as np
import pymargo
from pymargo.core import Engine
import pymargo.bulk as bulk
def call_rpc_on(engine, rpc_id, addr_str, provider_id, array_str):
addr = engine.lookup(addr_str)
handle = engine.create_handle(addr, rpc_id)
return handle.forward(provider_id, array_str)
with Engine('tcp', mode=pymargo.client) as engine:
rpc_id = engine.register("send_array")
myArray = np.random.rand(5, 7)
print(myArray)
blk = engine.create_bulk(myArray, bulk.read_write)
print("bulk created successfuly")
s = blk.to_base64()
print("bulk converted to base64: " + s)
call_rpc_on(engine, rpc_id, sys.argv[1], int(sys.argv[2]), s)
del blk
print("Shard {} has the size of {}.".format(i, col.size))
del pdb
del col
del address
del client
if __name__ == '__main__':
argc = len(sys.argv)
path = None
if (argc < 2):
path = ""
else:
path = sys.argv[1] + "/"
with Engine('ofi+tcp', mode=pymargo.server, use_progress_thread=False) \
as engine:
provider = SonataProvider(engine, 0)
addr = str(engine.addr())
print(addr)
admin = SonataAdmin(engine)
unqlite_files = glob.glob(path + '*.unqlite')
# extract number as index
ids = [int(f.split('.')[-2]) for f in unqlite_files]
# sort as numeric values
inds = sorted(range(len(ids)), key=lambda k: ids[k])
files = [unqlite_files[i] for i in inds] # files in correct order
for i, f in enumerate(files):
pdb_name = 'provdb.' + str(i)
admin.attach_database(addr, 0, pdb_name, 'unqlite',
"{ \"path\" : \"%s\" }" % f)
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
import sys
from pymargo.core import Engine
from pybake.target import BakeRegionID
from pybake.client import *
import numpy as np
mid = Engine('ofi+tcp')
server_addr = sys.argv[1]
mplex_id = int(sys.argv[2])
client = BakeClient(mid)
addr = mid.lookup(server_addr)
ph = client.create_provider_handle(addr, mplex_id)
# Testing get_eager_limit
lim = ph.get_eager_limit()
print "Eager limit is: " + str(lim)
# probe the provider handle (for all targets)
targets = ph.probe()
print "Probe found the following targets:"
for t in targets:
print "===== " + str(t)
target = targets[0]
# write into a region
arr = np.random.randn(5, 6)
class ProvDB():
def __init__(self, pdb_path='', pdb_sharded_num=0, pdb_addr=''):
if pdb_addr == '': # Standalone mode, need to create engine provider
self.pdb_engine = Engine('ofi+tcp', mode=pymargo.server)
self.pdb_provider = SonataProvider(self.pdb_engine, 0)
self.pdb_address = str(self.pdb_engine.addr())
self.pdb_admin = SonataAdmin(self.pdb_engine)
self.pdb_client = SonataClient(self.pdb_engine)
if pdb_path and int(pdb_sharded_num) > 0:
for i in range(int(pdb_sharded_num)):
pdb_name = 'provdb.' + str(i)
file_name = pdb_path + pdb_name + '.unqlite'
self.pdb_admin.attach_database(
self.pdb_address, 0, pdb_name, 'unqlite',
"{ \"path\" : \"%s\" }" % file_name)
else: # Other Chimbuko module created the engine
self.pdb_engine = Engine(pdb_addr.split(':')[0], pymargo.client)
self.pdb_address = pdb_addr
self.pdb_client = SonataClient(self.pdb_engine)
self.pdb_databases = []
self.pdb_collections = []
self.pdb_names = []
for i in range(pdb_sharded_num):
pdb_name = 'provdb.' + str(i)
self.pdb_names.append(pdb_name)
database = self.pdb_client.open(self.pdb_address, 0, pdb_name)
self.pdb_databases.append(database)
col = database.open('anomalies')
self.pdb_collections.append(col)
# print("=-=-=-=-=Initiated ProvDB instance {}=-=-=-=-=".format(
# self.pdb_address))
def __del__(self):
if self.pdb_databases:
for database in self.pdb_databases:
del database
database = None
del self.pdb_databases
self.pdb_databases = []
if self.pdb_collections:
for col in self.pdb_collections:
del col
col = None
del self.pdb_collections
self.pdb_collections = []
if self.pdb_names:
for name in self.pdb_names:
if hasattr(self, 'pdb_admin'):
self.pdb_admin.detach_database(self.pdb_address, 0, name)
del name
name = None
self.pdb_names = []
if self.pdb_client:
del self.pdb_client
self.pdb_client = None
if self.pdb_address:
del self.pdb_address
self.pdb_address = None
if hasattr(self, 'pdb_admin') and self.pdb_admin:
del self.pdb_admin
self.pdb_admin = None
if hasattr(self, 'pdb_provider') and self.pdb_provider:
del self.pdb_provider
self.pdb_provider = None
if self.pdb_engine:
self.pdb_engine.finalize()
gc.collect()
del self.pdb_engine
self.pdb_engine = None
def setUpClass(cls):
cls._engine = Engine('tcp://localhost:1234')
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
import sys
sys.path.append('.')
sys.path.append('build/lib.linux-x86_64-3.7')
from pymargo.core import Engine
from pybake.target import BakeRegionID
from pybake.client import *
mid = Engine('ofi+tcp')
def test():
server_addr = sys.argv[1]
mplex_id = int(sys.argv[2])
client = BakeClient(mid)
addr = mid.lookup(server_addr)
ph = client.create_provider_handle(addr, mplex_id)
# Testing get_eager_limit
lim = ph.get_eager_limit()
print("Eager limit is: "+str(lim))
# probe the provider handle (for all targets)
targets = ph.probe()
print("Probe found the following targets:")
for t in targets:
print("===== "+str(t))
target = targets[0]
from pymargo.core import Engine
if __name__ == '__main__':
with Engine('ofi+tcp') as engine:
engine.finalize()
print('Hello World')
class SeerClient:
def __init__(self, transport, address, p_id, db_n):
self.engine = Engine(transport)
self.server_addr = address
self.provider_id = p_id
self.db_name = db_n
# init the engine
self.client = SDSKVClient(self.engine)
self.addr = self.engine.lookup(self.server_addr)
self.provider_handle = self.client.create_provider_handle(self.addr, self.provider_id)
self.db = self.provider_handle.open(self.db_name)
# create a dictionary for values
self.command_dic = {}
self.used_unique_ids = []
self.count = 0
self.addingKeys = False
# Generate Hash
ip_addr = self.__getIP()
self.hash_val = self.__generateHash(ip_addr)
# excluded keys
self.removed_keys = []
def __getIP(self):
hostname = socket.gethostname()
IPAddr = socket.gethostbyname(hostname)
return str(IPAddr)
def __generateHash(self, value):
hash_object = hashlib.md5(value.encode())
temp_str = str(hash_object.hexdigest())
return ( temp_str[0:8] )
# Some Utility functions
def put_keyval(self, key, val):
self.db.put(key, val)
def del_key(self, key):
if (self.db.exists(key)):
self.db.erase(key)
else:
return "key " + key +" does not exist!"
def get_val(self, key):
if (self.db.exists(key)):
return self.db.get(key)
else:
return "key " + key +" does not exist!"
def exists(self, key):
return self.db.exists(key)
def list_keyVal(self):
key,val = self.db.list_keyvals()
return key,valvlist_keyvals
def shutdown(self):
self.client.shutdown_service(self.addr)
# User Operations
def list_keys(self):
"""List my keys and general ones"""
keys,vals = self.list_keyVal()
my_keys = []
for k in keys:
if k.startswith(self.hash_val) or k.startswith("00000000"):
if k not in self.removed_keys:
my_keys.append(k)
return my_keys
def list_my_keys_only(self):
"""List my keys only"""
key,val = self.list_keyVal()
my_keys = []
for k in key:
if k.startswith(self.hash_val): #my keys only
if k not in self.removed_keys:
my_keys.append(k)
return my_keys
def get_value_at_ts(self, key, timestep):
num_ranks = self.get_val("numRanks")
values = []
for r in range( int(num_ranks) ):
temp_key = self.hash_val + "@" + key + "#" + str(r) + "|" + str(timestep)
values.append( self.get_val(temp_key) )
return values
def set(self, key, value):
"""Adds a command"""
# Format of keys:
# NEW_KEY@HASH
# HASH@Type:Command%count
self.command_dic[key + "%" + str(self.count)] = value
self.count = self.count + 1
# Append to the list of removed keys
if key == "DEL":
self.removed_keys.append(value)
def commit(self):
"""Send commands to database"""
new_key = "NEW_KEY@" + self.hash_val
# Loop until you can add a new key
while ( self.db.exists(new_key) ):
continue
# Add keys to mochi DB
key_list = []
self.addingKeys = True
for key,val in self.command_dic.items():
key_padded = self.hash_val + "@" + key # add unique tag to each
self.db.put(key_padded, val) # put key in mochi db
key_list.append(key) # save the key for later deletion
print("key: ", key_padded)
print("val: ", val)
self.db.put(new_key, self.hash_val) # adding keys done
self.addingKeys = False
# Delete keys
for k in key_list: # delete keys that have been added to db
del self.command_dic[k]
