def __init__(self, appname, server_port, pull_call_back, push_call_back, confirm_pull_req, instrument_q):
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketManager" % appname)
# Number of workers in pool for connection
self.worker_count = MAX_THREADPOOL_WORKERS
# Call back function to process incoming pull requests.
self.pull_call_back = pull_call_back
# Call back function to process incoming push requests.
self.push_call_back = push_call_back
# Call back function to confirm incoming pull requests to dataframe.
self.confirm_pull_req = confirm_pull_req
# The socket that can be used by children.
self.sync_socket = self.setup_socket(server_port)
# The socket details.
addr, port = self.sync_socket.getsockname()
self.port = (addr if addr != "0.0.0.0" else "127.0.0.1", port)
super().__init__()
self.daemon = True
self.instrument_record = instrument_q
# Need to have a per app RLock, so that the app that just pushed data,
# does not ask for data that has not completely been pushed in.
# Potential memory leak?
# Another way to do it would be to keep persistent sockets.
# We'll see that later.
self.app_lock = dict()
def __init__(self, appname, server_port, pull_call_back, push_call_back, confirm_pull_req, instrument_q):
self.appname = appname
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketManager" % appname)
# Call back function to process incoming pull requests.
self.pull_call_back = pull_call_back
# Call back function to process incoming push requests.
self.push_call_back = push_call_back
# Call back function to confirm incoming pull requests to dataframe.
self.confirm_pull_req = confirm_pull_req
# The socket that can be used by children.
self.sync_socket = self.setup_socket(server_port)
# The socket details.
addr, port = self.sync_socket.getsockname()
self.shutdown = False
self.port = (addr if addr != "0.0.0.0" else "127.0.0.1", port)
self.instrument_record = instrument_q
self.clients = set()
super().__init__()
self.daemon = True
def __init__(self, appname, server_port, pull_call_back, push_call_back,
confirm_pull_req, instrument_q):
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketManager" % appname)
# Number of workers in pool for connection
self.worker_count = MAX_THREADPOOL_WORKERS
# Call back function to process incoming pull requests.
self.pull_call_back = pull_call_back
# Call back function to process incoming push requests.
self.push_call_back = push_call_back
# Call back function to confirm incoming pull requests to dataframe.
self.confirm_pull_req = confirm_pull_req
# The socket that can be used by children.
self.sync_socket = self.setup_socket(server_port)
# The socket details.
addr, port = self.sync_socket.getsockname()
self.port = (addr if addr != "0.0.0.0" else "127.0.0.1", port)
super().__init__()
self.daemon = True
self.instrument_record = instrument_q
# Need to have a per app RLock, so that the app that just pushed data,
# does not ask for data that has not completely been pushed in.
# Potential memory leak?
# Another way to do it would be to keep persistent sockets.
# We'll see that later.
self.app_lock = dict()
def __init__(self, appname, server_port, pull_call_back, push_call_back,
confirm_pull_req, instrument_q):
self.appname = appname
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketManager" % appname)
# Call back function to process incoming pull requests.
self.pull_call_back = pull_call_back
# Call back function to process incoming push requests.
self.push_call_back = push_call_back
# Call back function to confirm incoming pull requests to dataframe.
self.confirm_pull_req = confirm_pull_req
# The socket that can be used by children.
self.sync_socket = self.setup_socket(server_port)
# The socket details.
addr, port = self.sync_socket.getsockname()
self.shutdown = False
self.port = (addr if addr != "0.0.0.0" else "127.0.0.1", port)
self.instrument_record = instrument_q
self.clients = set()
super().__init__()
self.daemon = True
def __init__(self, appname, server_port, address, client_sock,
pull_call_back, push_call_back, confirm_pull_req,
delete_client, instrument_q):
self.logger = utils.get_logger("%s_SocketManager" % appname)
# Call back function to process incoming pull requests.
self.pull_call_back = pull_call_back
# Call back function to process incoming push requests.
self.push_call_back = push_call_back
# Call back function to confirm incoming pull requests to dataframe.
self.confirm_pull_req = confirm_pull_req
# The socket that can be used by children.
self.server_port = server_port
self.port = server_port
self.address = address
self.client_sock = client_sock
self.delete_client = delete_client
self.instrument_record = instrument_q
super().__init__()
self.daemon = True
def __init__(
self, appname, server_port, address, client_sock,
pull_call_back, push_call_back, confirm_pull_req, delete_client, instrument_q):
self.logger = utils.get_logger("%s_SocketManager" % appname)
# Call back function to process incoming pull requests.
self.pull_call_back = pull_call_back
# Call back function to process incoming push requests.
self.push_call_back = push_call_back
# Call back function to confirm incoming pull requests to dataframe.
self.confirm_pull_req = confirm_pull_req
# The socket that can be used by children.
self.server_port = server_port
self.port = server_port
self.address = address
self.client_sock = client_sock
self.delete_client = delete_client
self.instrument_record = instrument_q
super().__init__()
self.daemon = True
def __init__(self, appname, parent, details, types, version_by, instrument_q):
self.appname = appname
self.parent = parent
self.details = details
self.types = types
self.parent_version = None
if version_by == enums.VersionBy.FULLSTATE:
self.parent_version = "ROOT"
elif version_by == enums.VersionBy.TYPE:
self.parent_version = {
tp.__r_meta__.name: "ROOT"
for tp in types
}
elif version_by == enums.VersionBy.OBJECT_NOSTORE:
self.parent_version = {
tp.__r_meta__.name: dict()
for tp in types
}
else:
raise NotImplementedError()
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketConnector" % self.appname)
self.version_by = version_by
self.loop = asyncio.new_event_loop()
self.instrument_record = instrument_q
self.reader, self.writer = (
self.loop.run_until_complete(
create_connection(self.loop, self.parent))
if self.parent else
(None, None))
def __init__(self, appname, parent, details, types, instrument_q):
self.appname = appname
self.details = details
self.parent = parent
self.parent_version = None
self.instrument_record = instrument_q
self.parent_version = "ROOT"
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketConnector" % self.appname)
def __init__(self, appname, types, dump_graph):
self.types = types
self.type_map = {tp.__r_meta__.name: tp for tp in types}
self.version_graph = {tp.__r_meta__.name: dict() for tp in types}
self.state_to_app = {tp.__r_meta__.name: dict() for tp in types}
self.app_to_state = {tp.__r_meta__.name: dict() for tp in types}
self.logger = utils.get_logger("%s_ObjectVersionManagerVersionSent" %
appname)
self.dump_graphs = dump_graph
def __init__(self, appname, types, dump_graph, instrument_record):
self.types = types
self.type_map = {tp.__r_meta__.name: tp for tp in types}
self.version_graph = Graph()
self.state_to_app = dict()
self.app_to_state = dict()
self.logger = utils.get_logger("%s_FullStateVersionManager" % appname)
self.dump_graphs = dump_graph
self.instrument_record = instrument_record
def __init__(self,
appname,
types,
details=None,
server_port=0,
connection_as=enums.ConnectionStyle.TSocket,
instrument=None,
dump_graph=None,
resolver=None,
autoresolve=enums.AutoResolve.FullResolve,
mem_instrument=False):
self.appname = appname
self.logger = utils.get_logger("%s_Dataframe" % appname)
self.instrument = instrument
self.instrument_record = None
self._shutdown = False
if connection_as == enums.ConnectionStyle.TSocket:
SocketServer, SocketConnector = TSocketServer, TSocketConnector
elif connection_as == enums.ConnectionStyle.NPSocket:
SocketServer, SocketConnector = NPSocketServer, NPSocketConnector
elif connection_as == enums.ConnectionStyle.AIOSocket:
SocketServer, SocketConnector = AIOSocketServer, AIOSocketConnector
else:
raise NotImplementedError()
self.graph_change_event = Condition()
self.socket_server = SocketServer(self.appname, server_port,
self.fetch_call_back,
self.push_call_back,
self.confirm_fetch_req,
self.instrument_record)
self.socket_connector = SocketConnector(self.appname, details,
self.details, types,
self.instrument_record)
self.types = types
self.type_map = {tp.__r_meta__.name: tp for tp in self.types}
# THis is the local snapshot.
self.local_heap = ManagedHeap(types)
self.versioned_heap = None
# This is the dataframe's versioned graph.
self.versioned_heap = VersionManager(self.appname,
types,
resolver,
autoresolve,
instrument=instrument)
self.socket_server.start()
if self.socket_connector.has_parent_connection:
self.pull()
def __init__(
self, appname, types, details=None, server_port=0,
connection_as=enums.ConnectionStyle.TSocket,
instrument=None, dump_graph=None, resolver=None,
autoresolve=enums.AutoResolve.FullResolve):
self.appname = appname
self.logger = utils.get_logger("%s_Dataframe" % appname)
self.instrument = instrument
if self.instrument:
self.instrument_done = False
self.instrument_record = Queue()
self.instrument_writer = Process(target=self._save_instruments)
self.instrument_writer.daemon = True
self.instrument_writer.start()
else:
self.instrument_record = None
self._shutdown = False
if connection_as == enums.ConnectionStyle.TSocket:
SocketServer, SocketConnector = TSocketServer, TSocketConnector
elif connection_as == enums.ConnectionStyle.NPSocket:
SocketServer, SocketConnector = NPSocketServer, NPSocketConnector
elif connection_as == enums.ConnectionStyle.AIOSocket:
SocketServer, SocketConnector = AIOSocketServer, AIOSocketConnector
else:
raise NotImplementedError()
self.socket_server = SocketServer(
self.appname, server_port,
self.pull_call_back, self.push_call_back, self.confirm_pull_req,
self.instrument_record)
self.socket_connector = SocketConnector(
self.appname, details, self.details, types,
self.instrument_record)
self.types = types
self.type_map = {
tp.__r_meta__.name: tp for tp in self.types}
# THis is the local snapshot.
self.local_heap = ManagedHeap(types)
self.versioned_heap = None
# This is the dataframe's versioned graph.
self.versioned_heap = FullStateVersionManager(
self.appname, types, dump_graph,
self.instrument_record, resolver, autoresolve)
self.write_lock = RLock()
self.socket_server.start()
if self.socket_connector.has_parent_connection:
self.pull()
def __init__(self, appname, parent, details, types, instrument_q):
self.appname = appname
self.details = details
self.parent = parent
self.parent_version = None
self.parent_version = "ROOT"
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketConnector" % self.appname)
self.instrument_record = instrument_q
self.tpnames = {
tp.__r_meta__.name: tp.__r_meta__.name_chain
for tp in types
}
self.server_connection = self.connect_to_parent()
def __init__(self, appname, parent, details, types, instrument_q):
self.appname = appname
self.parent = parent
self.details = details
self.types = types
self.parent_version = None
self.parent_version = "ROOT"
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketConnector" % self.appname)
self.loop = asyncio.new_event_loop()
self.instrument_record = instrument_q
self.reader, self.writer = (
self.loop.run_until_complete(
create_connection(self.loop, self.parent))
if self.parent else
(None, None))
def __init__(self, appname, parent, details, types, version_by,
instrument_q):
self.appname = appname
self.details = details
self.parent = parent
self.parent_version = None
self.instrument_record = instrument_q
if version_by == enums.VersionBy.FULLSTATE:
self.parent_version = "ROOT"
elif version_by == enums.VersionBy.TYPE:
self.parent_version = {tp.__r_meta__.name: "ROOT" for tp in types}
elif version_by == enums.VersionBy.OBJECT_NOSTORE:
self.parent_version = {tp.__r_meta__.name: dict() for tp in types}
else:
raise NotImplementedError()
# Logger for SocketManager
self.logger = utils.get_logger("%s_SocketConnector" % self.appname)
self.version_by = version_by
def __init__(self,
appname,
types,
resolver=None,
autoresolve=AutoResolve.FullResolve,
instrument=None):
self.appname = appname
self.types = types
self.type_map = {tp.__r_meta__.name: tp for tp in types}
self.tpnames = {
tp.__r_meta__.name: tp.__r_meta__.name_chain
for tp in self.types
}
self.version_graph = Graph()
self.state_to_app = dict()
self.app_to_state = dict()
self.logger = utils.get_logger("%s_VersionManager" % appname)
self.resolver = resolver
self.autoresolve = autoresolve
self.write_lock = RLock()
self.instrument = instrument
def __init__(self,
appname,
types,
details=None,
server_port=0,
version_by=enums.VersionBy.FULLSTATE,
separate_dag=False,
connection_as=enums.ConnectionStyle.TSocket,
instrument=None,
dump_graph=None):
self.appname = appname
self.logger = utils.get_logger("%s_Dataframe" % appname)
self.version_by = version_by
self.instrument = instrument
if self.instrument:
self.instrument_done = False
self.instrument_record = Queue()
self.instrument_writer = Process(target=self._save_instruments)
self.instrument_writer.daemon = True
self.instrument_writer.start()
else:
self.instrument_record = None
self._shutdown = False
if connection_as == enums.ConnectionStyle.TSocket:
SocketServer, SocketConnector = TSocketServer, TSocketConnector
elif connection_as == enums.ConnectionStyle.NPSocket:
SocketServer, SocketConnector = NPSocketServer, NPSocketConnector
elif connection_as == enums.ConnectionStyle.AIOSocket:
SocketServer, SocketConnector = AIOSocketServer, AIOSocketConnector
else:
raise NotImplementedError()
self.socket_server = SocketServer(self.appname, server_port,
self.pull_call_back,
self.push_call_back,
self.confirm_pull_req,
self.instrument_record)
self.socket_connector = SocketConnector(self.appname, details,
self.details, types,
version_by,
self.instrument_record)
self.types = types
self.type_map = {tp.__r_meta__.name: tp for tp in self.types}
self.local_heap = ManagedHeap(types, version_by)
self.versioned_heap = None
if separate_dag:
self.versioned_heap = VersionManagerProcess(
self.appname, types, version_by)
self.versioned_heap.start()
elif version_by == enums.VersionBy.FULLSTATE:
self.versioned_heap = FullStateVersionManager(
self.appname, types, dump_graph, self.instrument_record)
elif version_by == enums.VersionBy.TYPE:
self.versioned_heap = TypeVersionManager(self.appname, types,
dump_graph)
elif version_by == enums.VersionBy.OBJECT_NOSTORE:
self.versioned_heap = ObjectVersionManagerVersionSent(
self.appname, types, dump_graph)
else:
raise NotImplementedError()
self.write_lock = RLock()
self.socket_server.start()
if self.socket_connector.has_parent_connection:
self.pull()
def __init__(self, appname, types, dump_graph=None):
self.types = types
self.type_map = {tp.__r_meta__.name: tp for tp in types}
self.logger = utils.get_logger("%s_VersionManager" % appname)
self.dump_graphs = dump_graph
