def send_end(self): L.info(f"send_end tagged_key:{self.tagged_key}") task_info = proxy_pb2.Task( taskId=self.name, model=proxy_pb2.Model(name=self.adapter.roll_site_header_string, dataKey=self.namespace)) command_test = proxy_pb2.Command(name="set_status") conf_test = proxy_pb2.Conf(overallTimeout=20000, completionWaitTimeout=20000, packetIntervalTimeout=20000, maxRetries=10) metadata = proxy_pb2.Metadata(task=task_info, src=self.topic_src, dst=self.topic_dst, command=command_test, operator="markEnd", seq=self.get_push_count(), ack=0) packet = proxy_pb2.Packet(header=metadata) try: # TODO:0: retry and sleep for all grpc call in RollSite self.stub.unaryCall(packet) except Exception as e: raise GrpcCallError('send_end', self.proxy_endpoint, e)
def send_end(self): L.info(f"RollSiteAdapter.send_end: tagged_key:{self.tagged_key}") task_info = proxy_pb2.Task(taskId=self.name, model=proxy_pb2.Model(name=self.adapter.roll_site_header_string, dataKey=self.namespace)) command_test = proxy_pb2.Command(name="set_status") conf_test = proxy_pb2.Conf(overallTimeout=20000, completionWaitTimeout=20000, packetIntervalTimeout=20000, maxRetries=10) metadata = proxy_pb2.Metadata(task=task_info, src=self.topic_src, dst=self.topic_dst, command=command_test, operator="markEnd", seq=self.push_batch_cnt, ack=0) packet = proxy_pb2.Packet(header=metadata) max_retry_cnt = 100 exception = None for i in range(1, max_retry_cnt + 1): try: # TODO:0: retry and sleep for all grpc call in RollSite self.stub.unaryCall(packet) exception = None break except Exception as e: L.info(f'caught exception in send_end for {self.name}, partition_id: {self.adapter.partition_id}: {e}. retrying. current retry count: {i}, max_retry_cnt: {max_retry_cnt}') exception = GrpcCallError('send_end', self.proxy_endpoint, e) if exception: raise exception
def push(self, obj): L.debug(f'pushing for task: {self.name}, partition id: {self.adapter.partition_id}, push cnt: {self.push_batch_cnt}') task_info = proxy_pb2.Task(taskId=self.name, model=proxy_pb2.Model(name=self.adapter.roll_site_header_string, dataKey=self.namespace)) command_test = proxy_pb2.Command() # TODO: conf test as config and use it conf_test = proxy_pb2.Conf(overallTimeout=200000, completionWaitTimeout=200000, packetIntervalTimeout=200000, maxRetries=10) metadata = proxy_pb2.Metadata(task=task_info, src=self.topic_src, dst=self.topic_dst, command=command_test, seq=0, ack=0) max_retry_cnt = 100 exception = None for i in range(1, max_retry_cnt + 1): try: self.stub.push(self.generate_message(obj, metadata)) exception = None self.push_batch_cnt += 1 break except Exception as e: exception = e L.info(f'caught exception in pushing {self.roll_site_header}, partition_id: {self.adapter.partition_id}: {e}. retrying. current retry count: {i}, max_retry_cnt: {max_retry_cnt}') time.sleep(min(5 * i, 30)) if exception: raise GrpcCallError("error in push", self.proxy_endpoint, exception)
def init_job_session_pair(self, roll_site_session_id, er_session_id): try: task_info = proxy_pb2.Task(model=proxy_pb2.Model( name=roll_site_session_id, dataKey=bytes(er_session_id, encoding='utf8'))) topic_src = proxy_pb2.Topic(name="init_job_session_pair", partyId=self.party_id, role=self.role, callback=None) topic_dst = proxy_pb2.Topic(name="init_job_session_pair", partyId=self.party_id, role=self.role, callback=None) command_test = proxy_pb2.Command(name="init_job_session_pair") conf_test = proxy_pb2.Conf(overallTimeout=1000, completionWaitTimeout=1000, packetIntervalTimeout=1000, maxRetries=10) metadata = proxy_pb2.Metadata(task=task_info, src=topic_src, dst=topic_dst, command=command_test, operator="init_job_session_pair", seq=0, ack=0) packet = proxy_pb2.Packet(header=metadata) self.stub.unaryCall(packet) L.info( f"send RollSiteContext init to Proxy: {to_one_line_string(packet)}" ) except Exception as e: raise GrpcCallError("init_job_session_pair", self.proxy_endpoint, e)
def wrap_grpc_packet(_json_body, _method, _url, _src_party_id, _dst_party_id, job_id=None, overall_timeout=DEFAULT_GRPC_OVERALL_TIMEOUT): _src_end_point = basic_meta_pb2.Endpoint(ip=IP, port=GRPC_PORT) _src = proxy_pb2.Topic(name=job_id, partyId="{}".format(_src_party_id), role=ROLE, callback=_src_end_point) _dst = proxy_pb2.Topic(name=job_id, partyId="{}".format(_dst_party_id), role=ROLE, callback=None) _task = proxy_pb2.Task(taskId=job_id) _command = proxy_pb2.Command(name=ROLE) _conf = proxy_pb2.Conf(overallTimeout=overall_timeout) _meta = proxy_pb2.Metadata(src=_src, dst=_dst, task=_task, command=_command, operator=_method, conf=_conf) _data = proxy_pb2.Data(key=_url, value=bytes(json.dumps(_json_body), 'utf-8')) return proxy_pb2.Packet(header=_meta, body=_data)
def pull(self, parties: list = None): self._pull_start_wall_time = time.time() self._pull_start_cpu_time = time.perf_counter() futures = [] for src_role, src_party_id in parties: src_party_id = str(src_party_id) roll_site_header = ErRollSiteHeader( roll_site_session_id=self.roll_site_session_id, name=self.name, tag=self.tag, src_role=src_role, src_party_id=src_party_id, dst_role=self.local_role, dst_party_id=self.party_id) _tagged_key = create_store_name(roll_site_header) name = _tagged_key model = proxy_pb2.Model(name=_stringify(roll_site_header)) task_info = proxy_pb2.Task(taskId=name, model=model) topic_src = proxy_pb2.Topic(name="get_status", partyId=src_party_id, role=src_role, callback=None) topic_dst = proxy_pb2.Topic(name="get_status", partyId=self.party_id, role=self.local_role, callback=None) get_status_command = proxy_pb2.Command(name="get_status") conf_test = proxy_pb2.Conf(overallTimeout=1000, completionWaitTimeout=1000, packetIntervalTimeout=1000, maxRetries=10) metadata = proxy_pb2.Metadata(task=task_info, src=topic_src, dst=topic_dst, command=get_status_command, operator="getStatus", seq=0, ack=0) packet = proxy_pb2.Packet(header=metadata) namespace = self.roll_site_session_id L.info( f"pulling prepared tagged_key: {_tagged_key}, packet:{to_one_line_string(packet)}" ) futures.append( RollSite.receive_exeutor_pool.submit(RollSite._thread_receive, self, packet, namespace, roll_site_header)) return futures