def initialize(self, check_commit):
"""
Initialize is used to initialize the graph for given url.
"""
print("just initialize")
self.graph = InitialGraph()
self.graph.main(self.url, self.api_doc, check_commit)
def initialize_graph(self) -> None:
"""Initialize the Graph on Redis based on ApiDoc
:param entrypoint_url: Entrypoint URL for the hydrus server
:return: None
"""
self.graph = InitialGraph()
self.redis_connection.delete("apigraph")
self.graph.main(self.entrypoint_url, self.api_doc, True)
self.redis_connection.sadd("fs:url", self.entrypoint_url)
class QueryFacades:
"""
It is used for call the above defined functions based on given query.
Using test as a bool which identify that function is using for tests.
"""
def __init__(self, api_doc, url, test):
self.endpoint_query = EndpointQuery()
self.api_doc = api_doc
self.url = url
self.properties = PropertiesQuery()
self.compare = CompareProperties()
self.test = test
self.redis_connection = RedisProxy()
self.connection = self.redis_connection.get_connection()
def initialize(self, check_commit):
"""
Initialize is used to initialize the graph for given url.
"""
print("just initialize")
self.graph = InitialGraph()
self.graph.main(self.url, self.api_doc, check_commit)
def check_fine_query(self, query):
if query.count(" ") != 1:
return RequestError("error")
def user_query(self, query):
"""
It calls function based on queries type.
"""
query = query.replace("show ", "")
if query == "endpoints":
data = self.endpoint_query.get_allEndpoints(query)
return data
elif query == "classEndpoints":
data = self.endpoint_query.get_classEndpoints(query)
return data
elif query == "collectionEndpoints":
data = self.endpoint_query.get_collectionEndpoints(query)
return data
elif "members" in query:
check_query = self.check_fine_query(query)
if isinstance(check_query, RequestError):
logger.info("Error: Incorrect query")
return None
else:
self.members = CollectionmembersQuery(self.api_doc, self.url,
self.graph)
if self.test:
data = self.members.data_from_server(
query.replace(" members", ""))
return data
else:
data = self.members.get_members(query)
return data
elif "objects" in query:
if query[-1] == " ":
logger.info("Error: incorrect query")
return None
check_query = self.check_fine_query(query)
if isinstance(check_query, RequestError):
logger.info("Error: Incorrect query")
return None
else:
data = self.properties.get_members_properties(query)
return data
elif "object" in query:
if query[-1] == " ":
logger.info("Error: incorrect query")
return None
check_query = self.check_fine_query(query)
if isinstance(check_query, RequestError):
logger.info("Error: Incorrect query")
return None
else:
data = self.properties.get_object_property(query)
return data
elif "Collection" in query:
if query[-1] == " ":
logger.info("Error: incorrect query")
return None
check_query = self.check_fine_query(query)
if isinstance(check_query, RequestError):
logger.info("Error: Incorrect query")
return None
else:
data = self.properties.get_collection_properties(query)
return data
elif "class" in query and "property_value" in query:
check_query = self.check_fine_query(query)
if isinstance(check_query, RequestError):
logger.info("Error: Incorrect query")
return None
else:
self.class_property = ClassPropertiesValue(
self.api_doc, self.url, self.graph)
data = self.class_property.get_property_value(query)
return data
elif "class" in query:
if query[-1] == " ":
logger.info("Error: incorrect query")
return None
check_query = self.check_fine_query(query)
if isinstance(check_query, RequestError):
logger.info("Error: Incorrect query")
return None
else:
data = self.properties.get_classes_properties(query)
return data
else:
if " and " in query or " or " in query:
if query[-1] == " " or query[-3] == "and" or query[-2] == "or":
logger.info("Error: incorrect query")
return None
query_len = len(query.split())
and_or_count = query.count("and") + query.count("or")
if query_len != (and_or_count + 2 * (and_or_count + 1)):
logger.info("Error: Incorrect query")
return None
data = self.compare.object_property_comparison_list(query)
return data
elif query.count(" ") == 1:
key, value = query.split(" ")
print("query: ", query)
search_index = "fs:" + key + ":" + value
for key in self.connection.keys():
if search_index == key.decode("utf8"):
data = self.connection.smembers(key)
return data
logger.info(
"Incorrect query: Use 'help' to know about querying format")
return None
class Agent(Session, socketio.ClientNamespace, socketio.Client):
"""Provides a straightforward GET, PUT, POST, DELETE -
CRUD interface - to query hydrus
"""
def __init__(self, entrypoint_url: str, namespace: str = '/sync') -> None:
"""Initialize the Agent
:param entrypoint_url: Entrypoint URL for the hydrus server
:param namespace: Namespace endpoint to listen for updates
:return: None
"""
self.entrypoint_url = entrypoint_url.strip().rstrip('/')
url_parse = urlparse(entrypoint_url)
self.entrypoint = url_parse.scheme + "://" + url_parse.netloc
self.api_name = url_parse.path.rstrip('/')
self.redis_proxy = RedisProxy()
self.redis_connection = self.redis_proxy.get_connection()
Session.__init__(self)
self.fetch_apidoc()
self.initialize_graph()
self.graph_operations = GraphOperations(self.entrypoint_url,
self.api_doc, self.redis_proxy)
# Declaring Socket Rules and instantiation Synchronization Socket
socketio.ClientNamespace.__init__(self, namespace)
socketio.Client.__init__(self, logger=True)
socketio.Client.register_namespace(self, self)
socketio.Client.connect(self,
self.entrypoint_url,
namespaces=namespace)
self.last_job_id = ""
def fetch_apidoc(self) -> HydraDoc:
"""Fetches API DOC from Link Header by checking the hydra apiDoc
relation and passes the obtained JSON-LD to doc_maker module of
hydra_python_core to return HydraDoc which is used by the agent.
:return HydraDoc created from doc_maker module
"""
try:
res = super().get(self.entrypoint_url)
api_doc_url = res.links[
'http://www.w3.org/ns/hydra/core#apiDocumentation']['url']
jsonld_api_doc = super().get(api_doc_url).json()
self.api_doc = doc_maker.create_doc(jsonld_api_doc,
self.entrypoint, self.api_name)
return self.api_doc
except:
print("Error parsing your API Documentation")
raise SyntaxError
def initialize_graph(self) -> None:
"""Initialize the Graph on Redis based on ApiDoc
:param entrypoint_url: Entrypoint URL for the hydrus server
:return: None
"""
self.graph = InitialGraph()
self.redis_connection.delete("apigraph")
self.graph.main(self.entrypoint_url, self.api_doc, True)
self.redis_connection.sadd("fs:url", self.entrypoint_url)
def get(self,
url: str = None,
resource_type: str = None,
follow_partial_links: bool = False,
filters: dict = {},
cached_limit: int = sys.maxsize) -> Union[dict, list, Paginator]:
"""READ Resource from Server/cached Redis
:param url: Resource URL to be fetched
:param resource_type: Resource object type
:param filters: filters to apply when searching, resources properties
:param cached_limit : Minimum amount of resources to be fetched
:param follow_partial_links: If set to True, Paginator can go through pages.
:return: Dict when one object or a list when multiple targeted objects
:return: Iterator when param follow_partial_links is set to true
Iterator will be returned.
Usage:
paginator = agent.get('http://*****:*****@type'] in self.api_doc.parsed_classes:
return response.json()
else:
if follow_partial_links:
return Paginator(response=response.json())
else:
return response.json()
else:
return response.text
def put(self, url: str, new_object: dict) -> Tuple[dict, str]:
"""CREATE resource in the Server/cache it on Redis
:param url: Server URL to create the resource at
:param new_object: Dict containing the object to be created
:return: Dict with server's response and resource URL
"""
response = super().put(url, json=new_object)
if response.status_code == 201:
url = response.headers['Location']
# Graph_operations returns the embedded resources if finding any
full_resource = super().get(url)
embedded_resources = self.graph_operations.put_processing(
url, full_resource.json())
self.process_embedded(embedded_resources)
return response.json(), url
else:
return response.text, ""
def post(self, url: str, updated_object: dict) -> dict:
"""UPDATE resource in the Server/cache it on Redis
:param url: Server URL to update the resource at
:param updated_object: Dict containing the updated object
:return: Dict with server's response
"""
response = super().post(url, json=updated_object)
if response.status_code == 200:
# Graph_operations returns the embedded resources if finding any
embedded_resources = \
self.graph_operations.post_processing(url, updated_object)
self.process_embedded(embedded_resources)
return response.json()
else:
return response.text
def delete(self, url: str) -> dict:
"""DELETE resource in the Server/delete it on Redis
:param url: Resource URL to be deleted
:return: Dict with server's response
"""
response = super().delete(url)
if response.status_code == 200:
self.graph_operations.delete_processing(url)
return response.json()
else:
return response.text
def process_embedded(self, embedded_resources: list) -> None:
"""Helper function to process a list of embedded resources
fetching and linking them to their parent Nodes
:param embedded_resources: List of dicts containing resources
"""
# Embedded resources are fetched and then properly linked
for embedded_resource in embedded_resources:
self.get(embedded_resource['embedded_url'])
self.graph_operations.link_resources(
embedded_resource['parent_id'],
embedded_resource['parent_type'],
embedded_resource['embedded_url'],
embedded_resource['embedded_type'], self.graph)
# Below are the functions that are responsible to process Socket Events
def on_connect(self, data: dict = None) -> None:
"""Method executed when the Agent is successfully connected to the Server
"""
if data:
self.last_job_id = data['last_job_id']
logger.info('Socket Connection Established - Synchronization ON')
def on_disconnect(self):
"""Method executed when the Agent is disconnected
"""
pass
def on_update(self, data) -> None:
"""Method executed when the Agent receives an event named 'update'
This is sent to all clients connected the server under the designed Namespace
:param data: Dict object with the last inserted row of modification's table
"""
row = data
# Checking if the Client is the last job id is up to date with the Server
if row['last_job_id'] == self.last_job_id:
# Checking if it's an already cached resource, if not it will ignore
if self.graph_operations.get_resource(row['resource_url']):
if row['method'] == 'POST':
self.graph_operations.delete_processing(
row['resource_url'])
self.get(row['resource_url'])
elif row['method'] == 'DELETE':
self.graph_operations.delete_processing(
row['resource_url'])
if row['method'] == 'PUT':
pass
# Updating the last job id
self.last_job_id = row['job_id']
# If last_job_id is not the same, there's more than one outdated modification
# Therefore the Client will try to get the diff of all modifications after his last job
else:
super().emit('get_modification_table_diff',
{'agent_job_id': self.last_job_id})
# Updating the last job id
self.last_job_id = row['job_id']
def on_modification_table_diff(self, data) -> None:
"""Event handler for when the client has to updated multiple rows
:param data: List with all modification rows to be updated
"""
new_rows = data
# Syncing procedure for every row received by mod table diff
for row in new_rows:
if self.graph_operations.get_resource(row['resource_url']):
if row['method'] == 'POST':
self.graph_operations.delete_processing(
row['resource_url'])
self.get(row['resource_url'])
elif row['method'] == 'DELETE':
self.graph_operations.delete_processing(
row['resource_url'])
if row['method'] == 'PUT':
pass
# Checking if the Agent is too outdated and can't be synced
if not new_rows:
logger.info('Server Restarting - Automatic Sync not possible')
self.initialize_graph()
# Agent should reply with a connect event with the last_job_id
super().emit('reconnect')
return None
# Updating the last job id
self.last_job_id = new_rows[0]['job_id']
def on_broadcast_event(self, data):
"""Method executed when the Agent receives a broadcast event
:param data: Object with the data broadcasted
"""
pass
class Agent(Session, socketio.ClientNamespace, socketio.Client):
"""Provides a straightforward GET, PUT, POST, DELETE -
CRUD interface - to query hydrus
"""
def __init__(self, entrypoint_url: str, namespace: str = '/sync') -> None:
"""Initialize the Agent
:param entrypoint_url: Entrypoint URL for the hydrus server
:param namespace: Namespace endpoint to listen for updates
:return: None
"""
self.entrypoint_url = entrypoint_url.strip().rstrip('/')
self.redis_proxy = RedisProxy()
self.redis_connection = self.redis_proxy.get_connection()
Session.__init__(self)
self.fetch_apidoc()
self.initialize_graph()
self.graph_operations = GraphOperations(self.entrypoint_url,
self.api_doc, self.redis_proxy)
# Declaring Socket Rules and instaciating Synchronization Socket
socketio.ClientNamespace.__init__(self, namespace)
socketio.Client.__init__(self, logger=True)
socketio.Client.register_namespace(self, self)
socketio.Client.connect(self,
self.entrypoint_url,
namespaces=namespace)
self.last_job_id = ""
def fetch_apidoc(self) -> dict:
if hasattr(self, 'api_doc'):
return self.api_doc
else:
jsonld_api_doc = super().get(self.entrypoint_url + '/vocab').json()
self.api_doc = doc_maker.create_doc(jsonld_api_doc)
return self.api_doc
def initialize_graph(self) -> None:
"""Initialize the Graph on Redis based on ApiDoc
:param entrypoint_url: Entrypoint URL for the hydrus server
:return: None
"""
self.graph = InitialGraph()
self.redis_connection.delete("apigraph")
self.graph.main(self.entrypoint_url, self.api_doc, True)
self.redis_connection.sadd("fs:url", self.entrypoint_url)
def get(self,
url: str = None,
resource_type: str = None,
filters: dict = {},
cached_limit: int = sys.maxsize) -> Union[dict, list]:
"""READ Resource from Server/cached Redis
:param url: Resource URL to be fetched
:param resource_type: Resource object type
:param filters: filters to apply when searching, resources properties
:param cached_limit : Minimum amount of resources to be fetched
:return: Dict when one object or a list when multiple targerted objects
"""
redis_response = self.graph_operations.get_resource(
url, resource_type, filters)
if redis_response:
if type(redis_response) is dict:
return redis_response
elif len(redis_response) >= cached_limit:
return redis_response
# If querying with resource type build url
# This can be more stable when adding Manages Block
# More on: https://www.hydra-cg.com/spec/latest/core/#manages-block
if resource_type:
url = self.entrypoint_url + "/" + resource_type + "Collection"
response = super().get(url, params=filters)
else:
response = super().get(url)
if response.status_code == 200:
# Graph_operations returns the embedded resources if finding any
embedded_resources = \
self.graph_operations.get_processing(url, response.json())
self.process_embedded(embedded_resources)
if response.json()['@type'] in self.api_doc.parsed_classes:
return response.json()
else:
return response.json()['members']
else:
return response.text
def put(self, url: str, new_object: dict) -> Tuple[dict, str]:
"""CREATE resource in the Server/cache it on Redis
:param url: Server URL to create the resource at
:param new_object: Dict containing the object to be created
:return: Dict with server's response and resource URL
"""
response = super().put(url, json=new_object)
if response.status_code == 201:
url = response.headers['Location']
# Graph_operations returns the embedded resources if finding any
embedded_resources = \
self.graph_operations.put_processing(url, new_object)
self.process_embedded(embedded_resources)
return response.json(), url
else:
return response.text, ""
def post(self, url: str, updated_object: dict) -> dict:
"""UPDATE resource in the Server/cache it on Redis
:param url: Server URL to update the resource at
:param updated_object: Dict containing the updated object
:return: Dict with server's response
"""
response = super().post(url, json=updated_object)
if response.status_code == 200:
# Graph_operations returns the embedded resources if finding any
embedded_resources = \
self.graph_operations.post_processing(url, updated_object)
self.process_embedded(embedded_resources)
return response.json()
else:
return response.text
def delete(self, url: str) -> dict:
"""DELETE resource in the Server/delete it on Redis
:param url: Resource URL to be deleted
:return: Dict with server's response
"""
response = super().delete(url)
if response.status_code == 200:
self.graph_operations.delete_processing(url)
return response.json()
else:
return response.text
def process_embedded(self, embedded_resources: list) -> None:
"""Helper function to process a list of embedded resources
fetching and linking them to their parent Nodes
:param embedded_resources: List of dicts containing resources
"""
# Embedded resources are fetched and then properly linked
for embedded_resource in embedded_resources:
self.get(embedded_resource['embedded_url'])
self.graph_operations.link_resources(
embedded_resource['parent_id'],
embedded_resource['parent_type'],
embedded_resource['embedded_url'])
# Below are the functions that are responsible to process Socket Events
def on_connect(self, data: dict = None) -> None:
"""Method executed when the Agent is successfully connected to the Server
"""
if data:
self.last_job_id = data['last_job_id']
logger.info('Socket Connection Established - Synchronization ON')
def on_disconnect(self):
"""Method executed when the Agent is disconnected
"""
pass
def on_update(self, data) -> None:
"""Method executed when the Agent receives an event named 'update'
This is sent to all clients connected the server under the designed Namespace
:param data: Dict object with the last inserted row of modification's table
"""
row = data
# Checking if the Client is the last job id is up to date with the Server
if row['last_job_id'] == self.last_job_id:
# Checking if it's an already cached resource, if not it will ignore
if self.graph_operations.get_resource(row['resource_url']):
if row['method'] == 'POST':
self.graph_operations.delete_processing(
row['resource_url'])
self.get(row['resource_url'])
elif row['method'] == 'DELETE':
self.graph_operations.delete_processing(
row['resource_url'])
if row['method'] == 'PUT':
pass
# Updating the last job id
self.last_job_id = row['job_id']
# If last_job_id is not the same, there's more than one outdated modification
# Therefore the Client will try to get the diff of all modifications after his last job
else:
super().emit('get_modification_table_diff',
{'agent_job_id': self.last_job_id})
# Updating the last job id
self.last_job_id = row['job_id']
def on_modification_table_diff(self, data) -> None:
"""Event handler for when the client has to updated multiple rows
:param data: List with all modification rows to be updated
"""
new_rows = data
# Syncing procedure for every row received by mod table diff
for row in new_rows:
if self.graph_operations.get_resource(row['resource_url']):
if row['method'] == 'POST':
self.graph_operations.delete_processing(
row['resource_url'])
self.get(row['resource_url'])
elif row['method'] == 'DELETE':
self.graph_operations.delete_processing(
row['resource_url'])
if row['method'] == 'PUT':
pass
# Checking if the Agent is too outdated and can't be synced
if not new_rows:
logger.info('Server Restarting - Automatic Sync not possible')
self.initialize_graph()
# Agent should reply with a connect event with the last_job_id
super().emit('reconnect')
return None
# Updating the last job id
self.last_job_id = new_rows[0]['job_id']
def on_broadcast_event(self, data):
"""Method executed when the Agent receives a broadcast event
:param data: Object with the data broadcasted
"""
pass