def send_ack(self, ack):
for client in self.clients:
if str(client["addr"]) == ack["addr"]:
Log.i(TAG, "Ack has send to client")
client["client"].send(json.dumps(ack["Ack"]).encode("UTF-8"))
def __init__(self):
self.loop = EventLoop()
self.isRunning: bool = self.shouldContinue()
Event.on('loop-rerun', self.run)
Log.i(TAG, "EventLoop Ready")
def __init__(self, host, port):
super().__init__(socket.AF_INET, socket.SOCK_STREAM)
self.port = port
self.host = host
self.clients = []
Log.d(TAG, f"Server is Ready")
def listen(self, **kwargs):
super().bind((self.host, self.port))
Log.i(TAG, f"Server is listening on port {self.port}")
Event.on('on_task_complete', self.send_ack)
while True:
super().listen(1)
client, addr = super().accept()
Log.d(TAG, f"Client connected with {addr}")
clientObj = {"client": client, "addr": addr}
self.clients.append(clientObj)
while True:
try:
raw_query = str(
client.recv(1024 * 1024 * 1024).decode("UTF-8"))
Log.d(TAG, f"Received data from client {addr}")
if raw_query.lower() == 'exit':
client.close()
break
json_query = json.loads(raw_query)
json_query['addr'] = str(addr)
if json_query['type'] is not None and json_query[
'type'] == 'Request':
Event.emmit('request', json.dumps(json_query))
Log.d(TAG, f"Client is requesting for RIDU operation")
elif json_query['type'] is not None and json_query[
'type'] == 'Subscription':
Log.d(TAG, f"Client is requesting for subscription")
Event.emmit('req_sub', json.dumps(json_query))
elif json_query['type'] is not None and json_query[
'type'] == 'Pipeline':
Log.d(TAG, f"Client is requesting for Data Pipeline")
Event.emmit('req_pipe', json_query)
elif json_query['type'] is not None and json_query[
'type'] == 'Provider':
Log.d(TAG,
f"Client is requesting for Provider Component")
Event.emmit('req_provider', json_query)
# code to communicate with hyperlite engine
except ConnectionResetError as err:
Log.e(TAG, f"Connection Reset -> {err}")
client.close()
Log.d(TAG, f"{self.clients}")
self.clients.remove(clientObj)
Log.i(TAG, "Client removed from Clients list")
Log.d(
TAG,
f"Connected clients -> {self.clients if len(self.clients) != 0 else 'No Clients'}"
)
break
except Exception as err:
Log.e(TAG, f"Connection broken -> {err}")
# errorSchema = """
# {
# "type": "Error",
# "message": "{}"
# }
# """.format(err)
# Log.d(TAG, errorSchema)
# client.send(errorSchema.encode('UTF-8'))
# client.close()
break
def get_collection(cls, col_name: str, db_name):
if Provider.collection_list.get(db_name) is not None:
database = Provider.collection_list.get(db_name)
result_col = None
for collection in database:
if col_name == collection.col_name:
result_col = collection
break
if result_col is not None:
Log.i(TAG, "Getting collection from Ram")
return result_col
else:
# Fetching or create new Collection
Log.i(TAG, "Fetching or create new Collection")
query = f"""
time_stamp,
db_name &eq "{db_name}",
col_name &eq "{col_name}"
"""
result = Provider.meta_collection.readOne(parser(query))
if not result:
Log.i(TAG, "Creating new collection")
return Provider.create_new_collection(col_name, db_name)
else:
Log.i(TAG, "Getting collection from disk")
result = loadCollection(
config.DATABASE_PATH + getPathSeparator() + str(result.get("time_stamp")) + ".col")
Provider.add_collection(result)
return result
else:
query = f"""
time_stamp,
db_name &eq "{db_name}",
col_name &eq "{col_name}"
"""
result = Provider.meta_collection.readOne(parser(query))
if not result:
Log.i(TAG, "Creating new collection")
return Provider.create_new_collection(col_name, db_name)
else:
Log.i(TAG, "Getting collection from disk")
result = loadCollection(
config.DATABASE_PATH + getPathSeparator() + str(result.get('time_stamp')) + ".col")
Provider.add_collection(result)
return result
def doctor():
if not os.path.exists(config.DATABASE_PATH):
Log.w(TAG, "Database directory does not exist")
os.makedirs(config.DATABASE_PATH)
Log.i(TAG, "Database directory Created")
def onPipeline(data):
Log.d(TAG, f"New data pipeline request - {data}")
loop_runner.loop.query_processes.put(DataPipelineProcess(data))
manage_loop_status()
def __init__(self, parsed_data):
self.data = parsed_data
Log.i(TAG, "InsertAllProcess created.")
def onRequest(data):
Log.d(TAG, f"New request - {data}")
loop_runner.loop.query_processes.put(
renderRIDUProcess(parsed_data=Parser.parse(data)))
manage_loop_status()
def onSubscription(data):
Log.d(TAG, f"New subscription request - {data}")
# TODO: Adding subscription plan
loop_runner.loop.subscriptions.put()
def manage_loop_status():
if not loop_runner.isRunning:
Log.i(TAG, "EventLoop is stopped, Rerunning EventLoop...")
Event.emmit('loop-rerun')
def initMe():
Log.welcome()
Log.c(TAG, "Starting Hyperlite Database")
Log.w(TAG, f"We are running on {config.PLATFORM} Operating System")
Log.i(TAG, f"Database files can be found on {config.DATABASE_PATH} ")
if os.path.exists(config.COLLECTION_PATH):
meta_col = loadCollection(config.COLLECTION_PATH)
Provider.meta_collection = meta_col
Log.i(TAG, "Meta collection found on disk")
else:
meta_col = Collection("hyperlite.col", "MetaData")
Provider.meta_collection = meta_col
Log.w(
TAG,
"Meta collection file not found so creating new meta collection")
def listen(self, **kwargs):
super().bind((self.host, self.port))
Log.i(TAG, f"Server is listening on port {self.port}")
Event.on('on_task_complete', self.send_ack)
while True:
super().listen(1)
client, addr = super().accept()
Log.d(TAG, f"Client connected with {addr}")
self.clients.append({
"client": client,
"addr": addr
})
while True:
try:
raw_query = str(client.recv(1024).decode("UTF-8"))
Log.d(TAG, f"Received data from client {addr}")
Log.d(TAG, f"Data -> {raw_query}")
if raw_query.lower() == 'exit':
client.close()
break
json_query = json.loads(raw_query)
json_query['addr'] = str(addr)
if json_query['type'] is not None and json_query['type'] == 'Request':
Event.emmit('request', json.dumps(json_query))
Log.d(TAG, f"Client is requesting for RIDU operation")
elif json_query['type'] is not None and json_query['type'] == 'Subscription':
Log.d(TAG, f"Client is requesting for subscription")
Event.emmit('req_sub', json.dumps(json_query))
# code to communicate with hyperlite engine
except Exception as err:
Log.e(TAG, f"Connection broken -> {err}")
client.close()
break
def __init__(self, parsed_data):
self.data = parsed_data
Log.i(TAG, "ReadByIdProcess created.")
def onProviderRequest(data):
Log.d(TAG, f"New provider request - {data}")
loop_runner.loop.query_processes.put(renderProviderProcess(data))
manage_loop_status()
def __init__(self, parsed_data):
self.data = parsed_data
Log.i(TAG, "UpdateProcess created.")
def __init__(self, data):
self.data = data
self.output = []
Log.i(TAG, "DataPipelineProcess created.")
