def listen(self, **kwargs):
super().bind((self.host, self.port))
Log.i(TAG, f"Server is listening on port {self.port}")
Event.on('on_task_complete', self.send_ack)
while True:
super().listen(1)
client, addr = super().accept()
Log.d(TAG, f"Client connected with {addr}")
self.clients.append({
"client": client,
"addr": addr
})
while True:
try:
raw_query = str(client.recv(1024).decode("UTF-8"))
Log.d(TAG, f"Received data from client {addr}")
Log.d(TAG, f"Data -> {raw_query}")
if raw_query.lower() == 'exit':
client.close()
break
json_query = json.loads(raw_query)
json_query['addr'] = str(addr)
if json_query['type'] is not None and json_query['type'] == 'Request':
Event.emmit('request', json.dumps(json_query))
Log.d(TAG, f"Client is requesting for RIDU operation")
elif json_query['type'] is not None and json_query['type'] == 'Subscription':
Log.d(TAG, f"Client is requesting for subscription")
Event.emmit('req_sub', json.dumps(json_query))
# code to communicate with hyperlite engine
except Exception as err:
Log.e(TAG, f"Connection broken -> {err}")
client.close()
break
def shouldContinue(self) -> bool:
Log.d(TAG,
f"Total query processes {self.loop.query_processes.qsize()}")
Log.d(TAG,
f"Total system processes {self.loop.system_process.qsize()}")
return (self.loop.query_processes.qsize() !=
0) or (self.loop.system_process.qsize() !=
0) or (self.loop.subscriptions.qsize() != 0)
def loadCollection(path):
Log.d(TAG, f"Getting a collection from {path}")
try:
return _pickle.load(open(path, 'rb'))
except Exception as ex:
Log.w(TAG, "Someone explicitly deleted the collection file from disk")
Log.e(TAG, f"Collection is not exist on {path}. {ex}")
return None
def exec(self):
Log.i(TAG, "Executing InsertProcess.")
db_name, col_name = BaseRIDUProcess.meta_separator(self.data.meta_data)
Log.d(TAG, f"{db_name, col_name}")
col = Collections.get_collection(col_name, db_name)
Log.d(TAG, "col obj fetched.")
acknowledgement = {
"Ack": col.insert(self.data.user_data),
"addr": self.data.addr
}
Event.emmit('col-change', col)
return acknowledgement
def __getCollectionNameForDisk(collection: Collection) -> str:
query = f" time_stamp,db_name &eq \"{collection.parent}\", col_name &eq \"{collection.col_name}\""
Log.d(TAG, "Searching collection name for disk")
data = Provider.meta_collection.readOne(parser.parser(query))
Log.d(TAG, f"Collection name for disk is {data}")
Log.d(TAG, f"{data.get('time_stamp')}.col")
return str(data.get("time_stamp"))
def writer(collection):
try:
doctor()
if collection is list:
for col in collection:
if col.col_name == config.DEFAULT_META_COLLECTION_NAME:
_pickle.dump(col, open(config.COLLECTION_PATH, "wb"))
Log.d(TAG, f"{collection.col_name} written on disk")
else:
_pickle.dump(col, open(__getNewCollectionUri(col), "wb"))
Log.d(TAG, f"{collection.col_name} written on disk")
else:
if collection.col_name == config.DEFAULT_META_COLLECTION_NAME:
_pickle.dump(collection, open(config.COLLECTION_PATH, "wb"))
Log.d(TAG, f"{collection.col_name} written on disk")
else:
_pickle.dump(collection,
open(__getNewCollectionUri(collection), "wb"))
Log.d(TAG, f"{collection.col_name} written on disk")
return True
except Exception as ex:
Log.e(TAG, f"Unable to write {collection.col_name} collection on disk")
return False
def onSubscription(data):
Log.d(TAG, f"New subscription request - {data}")
# TODO: Adding subscription plan
loop_runner.loop.subscriptions.put()
def onRequest(data):
Log.d(TAG, f"New request - {data}")
loop_runner.loop.query_processes.put(
renderRIDUProcess(parsed_data=Parser.parse(data)))
manage_loop_status()
def listen(self, **kwargs):
super().bind((self.host, self.port))
Log.i(TAG, f"Server is listening on port {self.port}")
Event.on('on_task_complete', self.send_ack)
while True:
super().listen(1)
client, addr = super().accept()
Log.d(TAG, f"Client connected with {addr}")
clientObj = {"client": client, "addr": addr}
self.clients.append(clientObj)
while True:
try:
raw_query = str(
client.recv(1024 * 1024 * 1024).decode("UTF-8"))
Log.d(TAG, f"Received data from client {addr}")
if raw_query.lower() == 'exit':
client.close()
break
json_query = json.loads(raw_query)
json_query['addr'] = str(addr)
if json_query['type'] is not None and json_query[
'type'] == 'Request':
Event.emmit('request', json.dumps(json_query))
Log.d(TAG, f"Client is requesting for RIDU operation")
elif json_query['type'] is not None and json_query[
'type'] == 'Subscription':
Log.d(TAG, f"Client is requesting for subscription")
Event.emmit('req_sub', json.dumps(json_query))
elif json_query['type'] is not None and json_query[
'type'] == 'Pipeline':
Log.d(TAG, f"Client is requesting for Data Pipeline")
Event.emmit('req_pipe', json_query)
elif json_query['type'] is not None and json_query[
'type'] == 'Provider':
Log.d(TAG,
f"Client is requesting for Provider Component")
Event.emmit('req_provider', json_query)
# code to communicate with hyperlite engine
except ConnectionResetError as err:
Log.e(TAG, f"Connection Reset -> {err}")
client.close()
Log.d(TAG, f"{self.clients}")
self.clients.remove(clientObj)
Log.i(TAG, "Client removed from Clients list")
Log.d(
TAG,
f"Connected clients -> {self.clients if len(self.clients) != 0 else 'No Clients'}"
)
break
except Exception as err:
Log.e(TAG, f"Connection broken -> {err}")
# errorSchema = """
# {
# "type": "Error",
# "message": "{}"
# }
# """.format(err)
# Log.d(TAG, errorSchema)
# client.send(errorSchema.encode('UTF-8'))
# client.close()
break
def __init__(self, host, port):
super().__init__(socket.AF_INET, socket.SOCK_STREAM)
self.port = port
self.host = host
self.clients = []
Log.d(TAG, f"Server is Ready")
def onProviderRequest(data):
Log.d(TAG, f"New provider request - {data}")
loop_runner.loop.query_processes.put(renderProviderProcess(data))
manage_loop_status()
def onPipeline(data):
Log.d(TAG, f"New data pipeline request - {data}")
loop_runner.loop.query_processes.put(DataPipelineProcess(data))
manage_loop_status()
def send_ack(self, ack):
Log.d(TAG, f"Query Task ack -> {ack}")
for client in self.clients:
if str(client["addr"]) == ack["addr"]:
Log.i(TAG, "Ack has send to client")
client["client"].send(json.dumps(ack["Ack"]).encode("UTF-8"))