def __init__(self, appname, server_port, pull_call_back, push_call_back, confirm_pull_req, instrument_q): # Logger for SocketManager self.logger = utils.get_logger("%s_SocketManager" % appname) # Number of workers in pool for connection self.worker_count = MAX_THREADPOOL_WORKERS # Call back function to process incoming pull requests. self.pull_call_back = pull_call_back # Call back function to process incoming push requests. self.push_call_back = push_call_back # Call back function to confirm incoming pull requests to dataframe. self.confirm_pull_req = confirm_pull_req # The socket that can be used by children. self.sync_socket = self.setup_socket(server_port) # The socket details. addr, port = self.sync_socket.getsockname() self.port = (addr if addr != "0.0.0.0" else "127.0.0.1", port) super().__init__() self.daemon = True self.instrument_record = instrument_q # Need to have a per app RLock, so that the app that just pushed data, # does not ask for data that has not completely been pushed in. # Potential memory leak? # Another way to do it would be to keep persistent sockets. # We'll see that later. self.app_lock = dict()
def __init__(self, appname, server_port, pull_call_back, push_call_back, confirm_pull_req, instrument_q): self.appname = appname # Logger for SocketManager self.logger = utils.get_logger("%s_SocketManager" % appname) # Call back function to process incoming pull requests. self.pull_call_back = pull_call_back # Call back function to process incoming push requests. self.push_call_back = push_call_back # Call back function to confirm incoming pull requests to dataframe. self.confirm_pull_req = confirm_pull_req # The socket that can be used by children. self.sync_socket = self.setup_socket(server_port) # The socket details. addr, port = self.sync_socket.getsockname() self.shutdown = False self.port = (addr if addr != "0.0.0.0" else "127.0.0.1", port) self.instrument_record = instrument_q self.clients = set() super().__init__() self.daemon = True
def __init__(self, appname, server_port, address, client_sock, pull_call_back, push_call_back, confirm_pull_req, delete_client, instrument_q): self.logger = utils.get_logger("%s_SocketManager" % appname) # Call back function to process incoming pull requests. self.pull_call_back = pull_call_back # Call back function to process incoming push requests. self.push_call_back = push_call_back # Call back function to confirm incoming pull requests to dataframe. self.confirm_pull_req = confirm_pull_req # The socket that can be used by children. self.server_port = server_port self.port = server_port self.address = address self.client_sock = client_sock self.delete_client = delete_client self.instrument_record = instrument_q super().__init__() self.daemon = True
def __init__( self, appname, server_port, address, client_sock, pull_call_back, push_call_back, confirm_pull_req, delete_client, instrument_q): self.logger = utils.get_logger("%s_SocketManager" % appname) # Call back function to process incoming pull requests. self.pull_call_back = pull_call_back # Call back function to process incoming push requests. self.push_call_back = push_call_back # Call back function to confirm incoming pull requests to dataframe. self.confirm_pull_req = confirm_pull_req # The socket that can be used by children. self.server_port = server_port self.port = server_port self.address = address self.client_sock = client_sock self.delete_client = delete_client self.instrument_record = instrument_q super().__init__() self.daemon = True
def __init__(self, appname, parent, details, types, version_by, instrument_q): self.appname = appname self.parent = parent self.details = details self.types = types self.parent_version = None if version_by == enums.VersionBy.FULLSTATE: self.parent_version = "ROOT" elif version_by == enums.VersionBy.TYPE: self.parent_version = { tp.__r_meta__.name: "ROOT" for tp in types } elif version_by == enums.VersionBy.OBJECT_NOSTORE: self.parent_version = { tp.__r_meta__.name: dict() for tp in types } else: raise NotImplementedError() # Logger for SocketManager self.logger = utils.get_logger("%s_SocketConnector" % self.appname) self.version_by = version_by self.loop = asyncio.new_event_loop() self.instrument_record = instrument_q self.reader, self.writer = ( self.loop.run_until_complete( create_connection(self.loop, self.parent)) if self.parent else (None, None))
def __init__(self, appname, parent, details, types, instrument_q): self.appname = appname self.details = details self.parent = parent self.parent_version = None self.instrument_record = instrument_q self.parent_version = "ROOT" # Logger for SocketManager self.logger = utils.get_logger("%s_SocketConnector" % self.appname)
def __init__(self, appname, types, dump_graph): self.types = types self.type_map = {tp.__r_meta__.name: tp for tp in types} self.version_graph = {tp.__r_meta__.name: dict() for tp in types} self.state_to_app = {tp.__r_meta__.name: dict() for tp in types} self.app_to_state = {tp.__r_meta__.name: dict() for tp in types} self.logger = utils.get_logger("%s_ObjectVersionManagerVersionSent" % appname) self.dump_graphs = dump_graph
def __init__(self, appname, types, dump_graph, instrument_record): self.types = types self.type_map = {tp.__r_meta__.name: tp for tp in types} self.version_graph = Graph() self.state_to_app = dict() self.app_to_state = dict() self.logger = utils.get_logger("%s_FullStateVersionManager" % appname) self.dump_graphs = dump_graph self.instrument_record = instrument_record
def __init__(self, appname, types, details=None, server_port=0, connection_as=enums.ConnectionStyle.TSocket, instrument=None, dump_graph=None, resolver=None, autoresolve=enums.AutoResolve.FullResolve, mem_instrument=False): self.appname = appname self.logger = utils.get_logger("%s_Dataframe" % appname) self.instrument = instrument self.instrument_record = None self._shutdown = False if connection_as == enums.ConnectionStyle.TSocket: SocketServer, SocketConnector = TSocketServer, TSocketConnector elif connection_as == enums.ConnectionStyle.NPSocket: SocketServer, SocketConnector = NPSocketServer, NPSocketConnector elif connection_as == enums.ConnectionStyle.AIOSocket: SocketServer, SocketConnector = AIOSocketServer, AIOSocketConnector else: raise NotImplementedError() self.graph_change_event = Condition() self.socket_server = SocketServer(self.appname, server_port, self.fetch_call_back, self.push_call_back, self.confirm_fetch_req, self.instrument_record) self.socket_connector = SocketConnector(self.appname, details, self.details, types, self.instrument_record) self.types = types self.type_map = {tp.__r_meta__.name: tp for tp in self.types} # THis is the local snapshot. self.local_heap = ManagedHeap(types) self.versioned_heap = None # This is the dataframe's versioned graph. self.versioned_heap = VersionManager(self.appname, types, resolver, autoresolve, instrument=instrument) self.socket_server.start() if self.socket_connector.has_parent_connection: self.pull()
def __init__( self, appname, types, details=None, server_port=0, connection_as=enums.ConnectionStyle.TSocket, instrument=None, dump_graph=None, resolver=None, autoresolve=enums.AutoResolve.FullResolve): self.appname = appname self.logger = utils.get_logger("%s_Dataframe" % appname) self.instrument = instrument if self.instrument: self.instrument_done = False self.instrument_record = Queue() self.instrument_writer = Process(target=self._save_instruments) self.instrument_writer.daemon = True self.instrument_writer.start() else: self.instrument_record = None self._shutdown = False if connection_as == enums.ConnectionStyle.TSocket: SocketServer, SocketConnector = TSocketServer, TSocketConnector elif connection_as == enums.ConnectionStyle.NPSocket: SocketServer, SocketConnector = NPSocketServer, NPSocketConnector elif connection_as == enums.ConnectionStyle.AIOSocket: SocketServer, SocketConnector = AIOSocketServer, AIOSocketConnector else: raise NotImplementedError() self.socket_server = SocketServer( self.appname, server_port, self.pull_call_back, self.push_call_back, self.confirm_pull_req, self.instrument_record) self.socket_connector = SocketConnector( self.appname, details, self.details, types, self.instrument_record) self.types = types self.type_map = { tp.__r_meta__.name: tp for tp in self.types} # THis is the local snapshot. self.local_heap = ManagedHeap(types) self.versioned_heap = None # This is the dataframe's versioned graph. self.versioned_heap = FullStateVersionManager( self.appname, types, dump_graph, self.instrument_record, resolver, autoresolve) self.write_lock = RLock() self.socket_server.start() if self.socket_connector.has_parent_connection: self.pull()
def __init__(self, appname, parent, details, types, instrument_q): self.appname = appname self.details = details self.parent = parent self.parent_version = None self.parent_version = "ROOT" # Logger for SocketManager self.logger = utils.get_logger("%s_SocketConnector" % self.appname) self.instrument_record = instrument_q self.tpnames = { tp.__r_meta__.name: tp.__r_meta__.name_chain for tp in types } self.server_connection = self.connect_to_parent()
def __init__(self, appname, parent, details, types, instrument_q): self.appname = appname self.parent = parent self.details = details self.types = types self.parent_version = None self.parent_version = "ROOT" # Logger for SocketManager self.logger = utils.get_logger("%s_SocketConnector" % self.appname) self.loop = asyncio.new_event_loop() self.instrument_record = instrument_q self.reader, self.writer = ( self.loop.run_until_complete( create_connection(self.loop, self.parent)) if self.parent else (None, None))
def __init__(self, appname, parent, details, types, version_by, instrument_q): self.appname = appname self.details = details self.parent = parent self.parent_version = None self.instrument_record = instrument_q if version_by == enums.VersionBy.FULLSTATE: self.parent_version = "ROOT" elif version_by == enums.VersionBy.TYPE: self.parent_version = {tp.__r_meta__.name: "ROOT" for tp in types} elif version_by == enums.VersionBy.OBJECT_NOSTORE: self.parent_version = {tp.__r_meta__.name: dict() for tp in types} else: raise NotImplementedError() # Logger for SocketManager self.logger = utils.get_logger("%s_SocketConnector" % self.appname) self.version_by = version_by
def __init__(self, appname, types, resolver=None, autoresolve=AutoResolve.FullResolve, instrument=None): self.appname = appname self.types = types self.type_map = {tp.__r_meta__.name: tp for tp in types} self.tpnames = { tp.__r_meta__.name: tp.__r_meta__.name_chain for tp in self.types } self.version_graph = Graph() self.state_to_app = dict() self.app_to_state = dict() self.logger = utils.get_logger("%s_VersionManager" % appname) self.resolver = resolver self.autoresolve = autoresolve self.write_lock = RLock() self.instrument = instrument
def __init__(self, appname, types, details=None, server_port=0, version_by=enums.VersionBy.FULLSTATE, separate_dag=False, connection_as=enums.ConnectionStyle.TSocket, instrument=None, dump_graph=None): self.appname = appname self.logger = utils.get_logger("%s_Dataframe" % appname) self.version_by = version_by self.instrument = instrument if self.instrument: self.instrument_done = False self.instrument_record = Queue() self.instrument_writer = Process(target=self._save_instruments) self.instrument_writer.daemon = True self.instrument_writer.start() else: self.instrument_record = None self._shutdown = False if connection_as == enums.ConnectionStyle.TSocket: SocketServer, SocketConnector = TSocketServer, TSocketConnector elif connection_as == enums.ConnectionStyle.NPSocket: SocketServer, SocketConnector = NPSocketServer, NPSocketConnector elif connection_as == enums.ConnectionStyle.AIOSocket: SocketServer, SocketConnector = AIOSocketServer, AIOSocketConnector else: raise NotImplementedError() self.socket_server = SocketServer(self.appname, server_port, self.pull_call_back, self.push_call_back, self.confirm_pull_req, self.instrument_record) self.socket_connector = SocketConnector(self.appname, details, self.details, types, version_by, self.instrument_record) self.types = types self.type_map = {tp.__r_meta__.name: tp for tp in self.types} self.local_heap = ManagedHeap(types, version_by) self.versioned_heap = None if separate_dag: self.versioned_heap = VersionManagerProcess( self.appname, types, version_by) self.versioned_heap.start() elif version_by == enums.VersionBy.FULLSTATE: self.versioned_heap = FullStateVersionManager( self.appname, types, dump_graph, self.instrument_record) elif version_by == enums.VersionBy.TYPE: self.versioned_heap = TypeVersionManager(self.appname, types, dump_graph) elif version_by == enums.VersionBy.OBJECT_NOSTORE: self.versioned_heap = ObjectVersionManagerVersionSent( self.appname, types, dump_graph) else: raise NotImplementedError() self.write_lock = RLock() self.socket_server.start() if self.socket_connector.has_parent_connection: self.pull()
def __init__(self, appname, types, dump_graph=None): self.types = types self.type_map = {tp.__r_meta__.name: tp for tp in types} self.logger = utils.get_logger("%s_VersionManager" % appname) self.dump_graphs = dump_graph