pickle_len = len(temp_pickle_string)
# running_end = (running_beg + 4000) if (pickle_len > (running_beg + 4000)) else pickle_len
for i in range(0, pickle_len, 4000):
j = (i + 4000) if ((i + 4000) < pickle_len) else pickle_len
pickle_arr.append(temp_pickle_string[i:j])
print(temp_super)
# print(len(list(temp_super.data)))
# print(repr(temp_string))
# with open("temp_string_holder.txt", "w") as file:
# file.write(temp_string)
# print(temp_pickle_string)
base_string = model_str
bash_bytes_string = model_str + "#Index"
i = 0
"""
for temp_string_iter in pickle_arr:
key_string = base_string + "#" + str(i)
value_bytes = LWWPairLattice(int(time.time()), temp_string_iter.encode())
client.put(key_string, value_bytes)
print(key_string)
# print(temp_string_iter)
# print("")
i += 1
i_string = str(i)
i_bytes = LWWPairLattice(int(time.time()), i_string.encode())
client.put(bash_bytes_string, i_bytes)
"""
i_two = int(
(((client.get(bash_bytes_string))[bash_bytes_string]).reveal()).decode())
class CloudburstConnection():
def __init__(self, func_addr, ip, tid=0, local=False):
'''
func_addr: The address of the Cloudburst interface, either localhost or
the address of an AWS ELB in cluster mode.
ip: The IP address of the client machine -- used to send and receive
responses.
tid: If multiple clients are running on the same machine, they will
need to use unique IDs.
local: A boolean representin whether the client is interacting with the
cluster in local or cluster mode.
'''
self.service_addr = 'tcp://' + func_addr + ':%d'
self.context = zmq.Context(1)
kvs_addr = self._connect()
# Picks a random offset of 10, mostly to alleviate port conflicts when
# running in local mode.
self.kvs_client = AnnaTcpClient(kvs_addr,
ip,
local=local,
offset=tid + 10)
self.func_create_sock = self.context.socket(zmq.REQ)
self.func_create_sock.connect(self.service_addr % FUNC_CREATE_PORT)
self.func_call_sock = self.context.socket(zmq.REQ)
self.func_call_sock.connect(self.service_addr % FUNC_CALL_PORT)
self.list_sock = self.context.socket(zmq.REQ)
self.list_sock.connect(self.service_addr % LIST_PORT)
self.dag_create_sock = self.context.socket(zmq.REQ)
self.dag_create_sock.connect(self.service_addr % DAG_CREATE_PORT)
self.dag_call_sock = self.context.socket(zmq.REQ)
self.dag_call_sock.connect(self.service_addr % DAG_CALL_PORT)
self.dag_delete_sock = self.context.socket(zmq.REQ)
self.dag_delete_sock.connect(self.service_addr % DAG_DELETE_PORT)
self.response_sock = self.context.socket(zmq.PULL)
response_port = 9000 + tid
self.response_sock.setsockopt(zmq.RCVTIMEO, 1000)
self.response_sock.bind('tcp://*:' + str(response_port))
self.response_address = 'tcp://' + ip + ':' + str(response_port)
self.rid = 0
def list(self, prefix=None):
'''
Returns a list of all the functions registered in the system.
prefix: An optional argument which, if specified, prunes the list of
returned functions to match the provided prefix.
'''
for fname in self._get_func_list(prefix):
print(fname)
def get_function(self, name):
'''
Retrieves a handle for an individual function. Returns None if the
function cannot be found in the system. The returned object can be
called like a regular Python function, which returns a CloudburstFuture.
name: The name of the function to retrieve.
'''
if name not in self._get_func_list():
print(f'''No function found with name {name}. To view all
functions, use the `list` method.''')
return None
return CloudburstFunction(name, self, self.kvs_client)
def register(self, function, name):
'''
Registers a new function or class with the system. The returned object
can be called like a regular Python function, which returns a Cloudburst
Future. If the input is a class, the class is expected to have a run
method, which is what is invoked at runtime.
function: The function object that we are registering.
name: A unique name for the function to be stored with in the system.
'''
func = Function()
func.name = name
func.body = serializer.dump(function)
self.func_create_sock.send(func.SerializeToString())
resp = GenericResponse()
resp.ParseFromString(self.func_create_sock.recv())
if resp.success:
registered_functon = CloudburstFunction(name, self,
self.kvs_client)
# print("55", self.kvs_client, "in register 66")
return registered_functon
else:
raise RuntimeError(
f'Unexpected error while registering function: {resp}.')
def register_dag(self, name, functions, connections):
'''
Registers a new DAG with the system. This operation will fail if any of
the functions provided cannot be identified in the system.
name: A unique name for this DAG.
functions: A list of names of functions to be included in this DAG.
connections: A list of ordered pairs of function names that represent
the edges in this DAG.
'''
flist = self._get_func_list()
for fname in functions:
if isinstance(fname, tuple):
fname = fname[0]
if fname not in flist:
raise RuntimeError(
f'Function {fname} not registered. Please register before '
+ 'including it in a DAG.')
dag = Dag()
dag.name = name
for function in functions:
ref = dag.functions.add()
if type(function) == tuple:
fname = function[0]
invalids = function[1]
ref.type = MULTIEXEC
else:
fname = function
invalids = []
ref.name = fname
for invalid in invalids:
ref.invalid_results.append(serializer.dump(invalid))
for pair in connections:
conn = dag.connections.add()
conn.source = pair[0]
conn.sink = pair[1]
self.dag_create_sock.send(dag.SerializeToString())
r = GenericResponse()
r.ParseFromString(self.dag_create_sock.recv())
return r.success, r.error
def call_dag(self,
dname,
arg_map,
direct_response=False,
consistency=NORMAL,
output_key=None,
client_id=None):
'''
Issues a new request to execute the DAG. Returns a CloudburstFuture that
dname: The name of the DAG to cexecute.
arg_map: A map from function names to lists of arguments for each of
the functions in the DAG.
direct_response: If True, the response will be synchronously received
by the client; otherwise, the result will be stored in the KVS.
consistency: The consistency mode to use with this function: either
NORMAL or MULTI.
output_key: The KVS key in which to store the result of thie DAG.
client_id: An optional ID associated with an individual client across
requests; this is used for causal metadata.
'''
dc = DagCall()
dc.name = dname
dc.consistency = consistency
if output_key:
dc.output_key = output_key
if client_id:
dc.client_id = client_id
for fname in arg_map:
fname_args = arg_map[fname]
if type(fname_args) != list:
fname_args = [fname_args]
args = [
serializer.dump(arg, serialize=False) for arg in fname_args
]
al = dc.function_args[fname]
al.values.extend(args)
if direct_response:
dc.response_address = self.response_address
self.dag_call_sock.send(dc.SerializeToString())
r = GenericResponse()
r.ParseFromString(self.dag_call_sock.recv())
if direct_response:
try:
result = self.response_sock.recv()
return serializer.load(result)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
return None
else:
raise e
else:
if r.success:
return CloudburstFuture(r.response_id, self.kvs_client,
serializer)
else:
return None
def delete_dag(self, dname):
'''
Removes the specified DAG from the system.
dname: The name of the DAG to delete.
'''
self.dag_delete_sock.send_string(dname)
r = GenericResponse()
r.ParseFromString(self.dag_delete_sock.recv())
return r.success, r.error
def get_object(self, key):
'''
Retrieves an arbitrary key from the KVS, automatically deserializes it,
and returns the value to the user.
'''
lattice = self.kvs_client.get(key)[key]
return serializer.load_lattice(lattice)
def put_object(self, key, value):
'''
Automatically wraps an object in a lattice and puts it into the
key-value store at the desired key.
'''
lattice = serializer.dump_lattice(value)
return self.kvs_client.put(key, lattice)
def exec_func(self, name, args):
call = FunctionCall()
call.name = name
call.request_id = self.rid
for arg in args:
argobj = call.arguments.values.add()
serializer.dump(arg, argobj)
self.func_call_sock.send(call.SerializeToString())
r = GenericResponse()
r.ParseFromString(self.func_call_sock.recv())
self.rid += 1
return r.response_id
def _connect(self):
sckt = self.context.socket(zmq.REQ)
sckt.connect(self.service_addr % CONNECT_PORT)
sckt.send_string('')
return sckt.recv_string()
def _get_func_list(self, prefix=None):
msg = prefix if prefix else ''
self.list_sock.send_string(msg)
flist = StringSet()
flist.ParseFromString(self.list_sock.recv())
return flist.keys
j = (i + 4000) if ((i + 4000) < pickle_len) else pickle_len
pickle_arr.append(temp_pickle_string[i:j])
print(temp_super)
# print(len(list(temp_super.data)))
# print(repr(temp_string))
# with open("temp_string_holder.txt", "w") as file:
# file.write(temp_string)
# print(temp_pickle_string)
base_string = model_str
bash_bytes_string = model_str + "#Index"
i = 0
for temp_string_iter in pickle_arr:
key_string = base_string + "#" + str(i)
value_bytes = LWWPairLattice(int(time.time()), temp_string_iter.encode())
client.put(key_string, value_bytes)
print(key_string)
# print(temp_string_iter)
# print("")
i += 1
i_string = str(i)
i_bytes = LWWPairLattice(int(time.time()), i_string.encode())
client.put(bash_bytes_string, i_bytes)
i_two = int((((client.get(bash_bytes_string))[bash_bytes_string]).reveal()).decode())
temp_pickle_ret = ""
for index in range(i_two):
temp_string_iter_two = pickle_arr[index]
key_string_two = base_string + "#" + str(index)
ret = (((client.get(key_string_two))[key_string_two]).reveal()).decode()
temp_pickle_ret = temp_pickle_ret + ret
