def __init__(self, http_host, http_port, http_port_retries, gcs_address, redis_address, redis_password, log_dir): self.health_check_thread: GCSHealthCheckThread = None self._gcs_rpc_error_counter = 0 # Public attributes are accessible for all head modules. # Walkaround for issue: https://github.com/ray-project/ray/issues/7084 self.http_host = "127.0.0.1" if http_host == "localhost" else http_host self.http_port = http_port self.http_port_retries = http_port_retries if use_gcs_for_bootstrap(): assert gcs_address is not None self.gcs_address = gcs_address else: self.redis_address = dashboard_utils.address_tuple(redis_address) self.redis_password = redis_password self.log_dir = log_dir self.aioredis_client = None self.aiogrpc_gcs_channel = None self.gcs_error_subscriber = None self.gcs_log_subscriber = None self.http_session = None self.ip = ray.util.get_node_ip_address() if not use_gcs_for_bootstrap(): ip, port = redis_address.split(":") else: ip, port = gcs_address.split(":") self.server = aiogrpc.server(options=(("grpc.so_reuseport", 0), )) grpc_ip = "127.0.0.1" if self.ip == "127.0.0.1" else "0.0.0.0" self.grpc_port = ray._private.tls_utils.add_port_to_grpc_server( self.server, f"{grpc_ip}:0") logger.info("Dashboard head grpc address: %s:%s", grpc_ip, self.grpc_port)
def start_outer_server(self, port: str = None) -> grpc_aio.Server: outer_server = grpc_aio.server(compression=self._default_compression) target_host = f'[::]:{port}' self.add_server_port(outer_server, target_host) logging.debug(f"outer target host = {target_host}") return outer_server
async def _serve(self, controller): ''' Server task ''' # Setup server self._server = aio.server() switchml_pb2_grpc.add_SessionServicer_to_server(self, self._server) switchml_pb2_grpc.add_SyncServicer_to_server(self, self._server) self._server.add_insecure_port('{}:{}'.format(self.ip, self.port)) # Lock to synchronize Barrier/Broadcast in case of reset self.lock = threading.RLock() ## Barrier # Incrementing operation id self._barrier_op_id = 0 # Worker counters and release events self._barrier_ctrs = {self._barrier_op_id: 0} self._barrier_events = {self._barrier_op_id: asyncio.Event()} ## Broadcast # Op values, bitmap and release events self._bcast_values = [] self._bcast_bitmap = [] self._bcast_events = [] # Controller self.ctrl = controller # Start gRPC server await self._server.start()
async def serve(): server = aio.server() helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) listen_addr = "[::]:50051" server.add_insecure_port(listen_addr) logging.info("Starting server on %s", listen_addr) await server.start() await server.wait_for_termination()
def __init__(self, node_ip_address, redis_address, dashboard_agent_port, gcs_address, redis_password=None, temp_dir=None, session_dir=None, runtime_env_dir=None, log_dir=None, metrics_export_port=None, node_manager_port=None, listen_port=0, object_store_name=None, raylet_name=None, logging_params=None): """Initialize the DashboardAgent object.""" # Public attributes are accessible for all agent modules. self.ip = node_ip_address if use_gcs_for_bootstrap(): assert gcs_address is not None self.gcs_address = gcs_address else: self.redis_address = dashboard_utils.address_tuple(redis_address) self.redis_password = redis_password self.aioredis_client = None self.gcs_address = None self.temp_dir = temp_dir self.session_dir = session_dir self.runtime_env_dir = runtime_env_dir self.log_dir = log_dir self.dashboard_agent_port = dashboard_agent_port self.metrics_export_port = metrics_export_port self.node_manager_port = node_manager_port self.listen_port = listen_port self.object_store_name = object_store_name self.raylet_name = raylet_name self.logging_params = logging_params self.node_id = os.environ["RAY_NODE_ID"] # TODO(edoakes): RAY_RAYLET_PID isn't properly set on Windows. This is # only used for fate-sharing with the raylet and we need a different # fate-sharing mechanism for Windows anyways. if sys.platform not in ["win32", "cygwin"]: self.ppid = int(os.environ["RAY_RAYLET_PID"]) assert self.ppid > 0 logger.info("Parent pid is %s", self.ppid) self.server = aiogrpc.server(options=(("grpc.so_reuseport", 0), )) grpc_ip = "127.0.0.1" if self.ip == "127.0.0.1" else "0.0.0.0" self.grpc_port = ray._private.tls_utils.add_port_to_grpc_server( self.server, f"{grpc_ip}:{self.dashboard_agent_port}") logger.info("Dashboard agent grpc address: %s:%s", grpc_ip, self.grpc_port) options = (("grpc.enable_http_proxy", 0), ) self.aiogrpc_raylet_channel = ray._private.utils.init_grpc_channel( f"{self.ip}:{self.node_manager_port}", options, asynchronous=True) self.http_session = None
async def serve(): server = aio.server() wedge_pb2_grpc.add_WedgeServicer_to_server(Wedge(), server) listen_addr = '[::]:50051' server.add_insecure_port(listen_addr) logging.info("Starting server on %s", listen_addr) await server.start() await server.wait_for_termination()
async def setUp(self): # Create async server self._server = aio.server(options=(('grpc.so_reuseport', 0), )) self._adhoc_handlers = AdhocGenericHandler() self._server.add_generic_rpc_handlers((self._adhoc_handlers, )) port = self._server.add_insecure_port('[::]:0') address = 'localhost:%d' % port await self._server.start() # Create async channel self._channel = aio.insecure_channel(address)
async def serve_async(): class Greeter(helloworld_pb2_grpc.GreeterServicer): async def SayHello(self, request, context): await sleep_async(DELAY_TIME) return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) server = aio.server() helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) server.add_insecure_port('[::]:50051') await server.start() await server.wait_for_termination()
def _create_server(self): self._servicer = InferenceServicer(self._data_plane) self._server = aio.server( ThreadPoolExecutor(max_workers=self._settings.grpc_workers)) add_GRPCInferenceServiceServicer_to_server(self._servicer, self._server) self._server.add_insecure_port( f"{self._settings.host}:{self._settings.grpc_port}") return self._server
async def main(port, log_storage="/home/tbjc1magic/log"): server = aio.server() task_manager = TaskManager() task_manager_thread = asyncio.to_thread(task_manager.run) data_collector_service_pb2_grpc.add_DataCollectorServicer_to_server( DataCollectorServicer(log_storage, task_manager), server) service_names = ( data_collector_service_pb2.DESCRIPTOR. services_by_name["DataCollector"].full_name, reflection.SERVICE_NAME, ) reflection.enable_server_reflection(service_names, server) server.add_insecure_port(f"[::]:{port}") await server.start() await asyncio.gather(server.wait_for_termination(), task_manager_thread)
async def __run(): global __server await __lock.acquire() __server = aio.server( ThreadPoolExecutor(max_workers=server_config.max_workers)) route_pb2_grpc.add_UpdateServerRouteServicer_to_server(Greeter(), __server) __server.add_insecure_port(f'{server_config.host}:{server_config.port}') logging.info("gRPC 启动中") await __server.start() logging.info("gRPC 已启动") # 等待停止信号 logging.info("启动 gRPC 进程运行阻塞锁(额外)") await __lock.acquire() logging.info("脱离 gRPC 进程运行阻塞锁(额外)") await __server.wait_for_termination() logging.info("脱离 gRPC 运行阻塞")
async def serve_order_book(port, data_collector_address="localhost:9999"): server = aio.server() order_book_manager = OrderBookManager( ["bnbbtc", "ethbusd", "dogeusdt"], save_data=True, data_collector_address=data_collector_address, server=server, ) order_book_service_pb2_grpc.add_OrderBookServicer_to_server( OrderBookServier(order_book_manager), server) service_names = ( order_book_service_pb2.DESCRIPTOR.services_by_name["OrderBook"]. full_name, reflection.SERVICE_NAME, ) reflection.enable_server_reflection(service_names, server) server.add_insecure_port(f"[::]:{port}") await server.start() await asyncio.gather(server.wait_for_termination(), order_book_manager.run())
def __init__( self, http_host, http_port, http_port_retries, gcs_address, log_dir, temp_dir, session_dir, minimal, ): self.minimal = minimal self.health_check_thread: GCSHealthCheckThread = None self._gcs_rpc_error_counter = 0 # Public attributes are accessible for all head modules. # Walkaround for issue: https://github.com/ray-project/ray/issues/7084 self.http_host = "127.0.0.1" if http_host == "localhost" else http_host self.http_port = http_port self.http_port_retries = http_port_retries self.gcs_address = None assert gcs_address is not None self.gcs_address = gcs_address self.log_dir = log_dir self.temp_dir = temp_dir self.session_dir = session_dir self.aiogrpc_gcs_channel = None self.gcs_error_subscriber = None self.gcs_log_subscriber = None self.ip = ray.util.get_node_ip_address() ip, port = gcs_address.split(":") self.server = aiogrpc.server(options=(("grpc.so_reuseport", 0), )) grpc_ip = "127.0.0.1" if self.ip == "127.0.0.1" else "0.0.0.0" self.grpc_port = ray._private.tls_utils.add_port_to_grpc_server( self.server, f"{grpc_ip}:0") logger.info("Dashboard head grpc address: %s:%s", grpc_ip, self.grpc_port) # If the dashboard is started as non-minimal version, http server should # be configured to expose APIs. self.http_server = None
async def serve(): port = os.getenv('SOCKS5_PORT', '9050') host = os.getenv('SOCKS5_HOST', '127.0.0.1') connector = create_tor_connector({ 'host': host, 'port': port, 'username': os.getenv('SOCKS5_USERNAME', ''), 'password': os.getenv('SOCKS5_PASSWORD', '') }) print("Attempting to connect to Tor on {host}:{port}".format(host=host, port=port)) async with aiohttp.ClientSession(connector=connector) as session: print('Tor connected.') server = aio.server() scrape_pb2_grpc.add_ScraperServicer_to_server(ScraperService(session), server) server.add_insecure_port("[::]:50051") print('Starting gRPC server on port 50051') await server.start() await server.wait_for_termination()
def __init__(self, address, port): super().__init__() self.server = aio.server() self.server.add_insecure_port(f'{address}:{port}')
def __init__(self): self.nodeid = 1 #configuration # args = config.config() # args.project_path = "./dataset/images" self.datapath = 'dataset' # data = ODMLoadDatasetStage(self.datapath , args, progress=5.0, # verbose=args.verbose) #distance to include the image in meters self.include_distance = 10 #run the dataset layer #data.run() print('sfm') print(self.datapath) #opensfm configuration opensfm_config = opensfm_interface.setup_opensfm_config(self.datapath) #extract metadata # camera_models = {} # current_path = '/home/j/ODM-master/grpc_stages/node1/' # ref_image = ['DJI_0019.JPG', 'DJI_0018.JPG','DJI_0020.JPG','DJI_0021.JPG','DJI_0022.JPG','DJI_0023.JPG','DJI_0024.JPG' # ,'DJI_0025.JPG','DJI_0026.JPG','DJI_0027.JPG','DJI_0028.JPG','DJI_0029.JPG','DJI_0030.JPG','DJI_0031.JPG','DJI_0032.JPG','DJI_0033.JPG','DJI_0034.JPG','DJI_0035.JPG'] # cand_images = ['DJI_0019.JPG', 'DJI_0018.JPG','DJI_0020.JPG','DJI_0021.JPG','DJI_0022.JPG','DJI_0023.JPG','DJI_0024.JPG' # ,'DJI_0025.JPG','DJI_0026.JPG','DJI_0027.JPG','DJI_0028.JPG','DJI_0029.JPG','DJI_0030.JPG','DJI_0031.JPG','DJI_0032.JPG','DJI_0033.JPG','DJI_0034.JPG','DJI_0035.JPG'] # for each in ref_image: # d = opensfm_interface.extract_metadata_image('/home/j/ODM-master/grpc_stages/node1/'+each, opensfm_config) # if d['camera'] not in camera_models: # camera = exif.camera_from_exif_metadata(d, opensfm_config) # camera_models[d['camera']] = camera # opensfm_interface.save_exif('/home/j/ODM-master/grpc_stages/node1/exif', each,d) # opensfm_interface.save_camera_models('/home/j/ODM-master/grpc_stages/node1/', camera_models) # #c = opensfm_interface.extract_metadata_image('/home/j/ODM-master/grpc_stages/node1/DJI_0019.JPG', opensfm_config) # print(d) # print(camera_models) # #opensfm_interface.save_exif('/home/j/ODM-master/grpc_stages/node1/', 'DJI_0019.JPG', c) # #save the exif metadata to file in a folder # # send extracted metedata and camera model back # #feature extraction # for each in ref_image: # opensfm_interface.detect(current_path+'features', current_path+each,each ,opensfm_config) # #opensfm_interface.detect(current_path+'features', current_path+'DJI_0019.JPG','DJI_0019.JPG' ,opensfm_config) # #feature matching # pairs_matches, preport = new_matching.match_images(current_path, ref_image, cand_images, opensfm_config) # new_matching.save_matches(current_path, ref_image, pairs_matches) # print('matching') # #create tracks first # features, colors = tracking.load_features(current_path+'features', ref_image, opensfm_config) # matches = tracking.load_matches(current_path, ref_image) # graph = tracking.create_tracks_graph(features, colors, matches, # opensfm_config) # opensfm_interface.save_tracks_graph(graph, current_path) # #reconstruction # # load tracks graph # graph = opensfm_interface.load_tracks_graph(current_path) # report, reconstructions = reconstruction.incremental_reconstruction(current_path, graph, opensfm_config) # opensfm_interface.save_reconstruction(current_path,reconstructions) # #opensfm_interface.save_report(io.json_dumps(report), 'reconstruction.json') # outputs = {} # photos = [] # from opendm import photo # from opendm import types # for each in ref_image: # photos += [types.ODM_Photo(current_path+each)] # # get match image sizes # outputs['undist_image_max_size'] = max( # gsd.image_max_size(photos, 5.0, current_path+'reconstruction.json'), # 0.1 # ) # print(outputs) # #undistort image dataset: # opensfm_interface.opensfm_undistort(current_path, opensfm_config) # #export visualsfm # opensfm_interface.open_export_visualsfm(current_path, opensfm_config) # #compute depthmaps # opensfm_interface.open_compute_depthmaps(current_path, opensfm_config) # #mve stage 1 makescene # #input compute depthmaps file # mve_file_path = '/home/j/ODM-master/grpc_stages/node1/mve' # nvm_file = '/home/j/ODM-master/grpc_stages/node1/undistorted/reconstruction.nvm' # mve_interface.mve_makescene(nvm_file, mve_file_path, 2) # #mve stage 2 dense reconstruction # mve_interface.mve_dense_recon(outputs['undist_image_max_size'], mve_file_path, 2) # #mve stage 3 scene2pset_path # mve_model = io.join_paths(mve_file_path, 'mve_dense_point_cloud.ply') # mve_interface.mve_scene2pset(mve_file_path, mve_model,outputs['undist_image_max_size'],2) # #mve stage 4 clean_mesh # mve_interface.mve_cleanmesh(0.6, mve_model, 2) # # filterpoint cloud # odm_filterpoints = '/home/j/ODM-master/grpc_stages/node1/filterpoints' # filterpoint_cloud = io.join_paths(odm_filterpoints, "point_cloud.ply") # filterpoint_interface.filter_points(odm_filterpoints, mve_model, filterpoint_cloud,2) # #meshing stage # odm_mesh_folder= '/home/j/ODM-master/grpc_stages/node1/mesh' # odm_mesh_ply = io.join_paths(odm_mesh_folder, "odm_mesh.ply") # mesh_interface.mesh_3d(odm_mesh_folder, odm_mesh_ply, filterpoint_cloud, 2) # #texturing stage # mvs_folder= '/home/j/ODM-master/grpc_stages/node1/mvs' # mvs_texturing.mvs_texturing(odm_mesh_ply, mvs_folder, nvm_file) #https://stackoverflow.com/questions/45071567/how-to-send-custom-header-metadata-with-python-grpc class Servicer(sendFile_pb2_grpc.FileServiceServicer): def __init__(self): self.dataset_dir = './dataset' self.tmp_file_name = './dataset2/IMG_2359.JPG' async def upload(self, request_iterator, context): # client uploads images to this node #request iterator is the file iterator through the chuncks #self.tmp_file_name is the name to save the file chucks to nodeid = '' filename = '' for key, value in context.invocation_metadata(): if (key == 'node-id'): nodeid = value #check if there is a dir for the node id system.mkdir_p(nodeid) if key == 'filename': filename = value print('Received initial metadata: key=%s value=%s' % (key, value)) #print(os.path.dirname(os.path.abspath(__file__))) if (nodeid != '' and filename != ''): save_chunks_to_file(request_iterator, nodeid + '/images/' + filename) return sendFile_pb2.UploadStatus( Message=" Successul ", Code=sendFile_pb2.UploadStatusCode.Ok) else: print('bad node id and bad filename') # reply = sendFile_pb2.UploadStatus() # reply.Message = " Failed " # reply.c return sendFile_pb2.UploadStatus( Message=" Failure ", Code=sendFile_pb2.UploadStatusCode.Failed) def download(self, request, context): if request.name: return get_file_chunks(self.tmp_file_name) def compute(self, request, context): taskName = request.taskName #metadata use image name #use node id #opensfm compute # detect feature # # if(taskName == "compute_image_feature"): # elif(taskName == 'compute_matching_two_images'):0 # elif(taskName == 'compute_'): # elif(taskName == ''): return 0 self.server = aio.server() #self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) sendFile_pb2_grpc.add_FileServiceServicer_to_server( Servicer(), self.server) neighbor_ip = ['50001', '50002'] self.has_neighbor = defaultdict(lambda: "Node not present.") # tuple (false as any response from neighbor, filelocation) neighbor_response = 0 #increment neighbors response as each neighbor respond for each in neighbor_ip: self.has_neighbor[each] = (False, "") self.leader = True
def __init__( self, node_ip_address, dashboard_agent_port, gcs_address, minimal, temp_dir=None, session_dir=None, runtime_env_dir=None, log_dir=None, metrics_export_port=None, node_manager_port=None, listen_port=0, object_store_name=None, raylet_name=None, logging_params=None, disable_metrics_collection: bool = False, ): """Initialize the DashboardAgent object.""" # Public attributes are accessible for all agent modules. self.ip = node_ip_address self.minimal = minimal assert gcs_address is not None self.gcs_address = gcs_address self.temp_dir = temp_dir self.session_dir = session_dir self.runtime_env_dir = runtime_env_dir self.log_dir = log_dir self.dashboard_agent_port = dashboard_agent_port self.metrics_export_port = metrics_export_port self.node_manager_port = node_manager_port self.listen_port = listen_port self.object_store_name = object_store_name self.raylet_name = raylet_name self.logging_params = logging_params self.node_id = os.environ["RAY_NODE_ID"] self.metrics_collection_disabled = disable_metrics_collection # TODO(edoakes): RAY_RAYLET_PID isn't properly set on Windows. This is # only used for fate-sharing with the raylet and we need a different # fate-sharing mechanism for Windows anyways. if sys.platform not in ["win32", "cygwin"]: self.ppid = int(os.environ["RAY_RAYLET_PID"]) assert self.ppid > 0 logger.info("Parent pid is %s", self.ppid) # Setup raylet channel options = ray_constants.GLOBAL_GRPC_OPTIONS self.aiogrpc_raylet_channel = ray._private.utils.init_grpc_channel( f"{self.ip}:{self.node_manager_port}", options, asynchronous=True) # Setup grpc server self.server = aiogrpc.server(options=(("grpc.so_reuseport", 0), )) grpc_ip = "127.0.0.1" if self.ip == "127.0.0.1" else "0.0.0.0" try: self.grpc_port = ray._private.tls_utils.add_port_to_grpc_server( self.server, f"{grpc_ip}:{self.dashboard_agent_port}") except Exception: # TODO(SongGuyang): Catch the exception here because there is # port conflict issue which brought from static port. We should # remove this after we find better port resolution. logger.exception( "Failed to add port to grpc server. Agent will stay alive but " "disable the grpc service.") self.server = None self.grpc_port = None else: logger.info("Dashboard agent grpc address: %s:%s", grpc_ip, self.grpc_port) # If the agent is started as non-minimal version, http server should # be configured to communicate with the dashboard in a head node. self.http_server = None # Used by the agent and sub-modules. # TODO(architkulkarni): Remove gcs_client once the agent exclusively uses # gcs_aio_client and not gcs_client. self.gcs_client = GcsClient(address=self.gcs_address) _initialize_internal_kv(self.gcs_client) assert _internal_kv_initialized() self.gcs_aio_client = GcsAioClient(address=self.gcs_address) self.publisher = GcsAioPublisher(address=self.gcs_address)