def GetRawDataLatestTimeStamp(self, request, context): response = dj_pb.RawDataResponse(timestamp=timestamp_pb2.Timestamp( seconds=0)) meta_status = self._check_data_source_meta(request.data_source_meta) if meta_status.code != 0: response.status.MergeFrom(meta_status) return response manifest_manager = self._fsm.get_mainifest_manager() ts = manifest_manager.get_raw_date_latest_timestamp( request.partition_id) if ts is not None: response.timestamp.MergeFrom(ts) return response
def RequestJoinPartition(self, request, context): response = dj_pb.RawDataResponse() meta_status = self._check_data_source_meta(request.data_source_meta) if meta_status.code != 0: response.status.MergeFrom(meta_status) return response rank_status = self._check_rank_id(request.rank_id) if rank_status.code != 0: response.status.MergeFrom(rank_status) return response data_source = self._fsm.get_data_source() if data_source.state != common_pb.DataSourceState.Processing: response.status.code = -3 response.status.error_message = \ "data source is not at processing state" else: manifest_manager = self._fsm.get_mainifest_manager() rank_id = request.rank_id manifest = None partition_id = None if request.partition_id < 0 \ else request.partition_id if request.HasField('sync_example_id'): manifest = manifest_manager.alloc_sync_exampld_id( rank_id, partition_id ) elif request.HasField('join_example'): manifest = manifest_manager.alloc_join_example( rank_id, partition_id ) else: response.status.code = -4 response.status.error_message = "request not support" if response.status.code == 0: if manifest is not None: response.manifest.MergeFrom(manifest) else: assert partition_id is None, \ "only the request without appoint partition "\ "support response no manifest" response.finished.MergeFrom(empty_pb2.Empty()) return response
def RequestJoinPartition(self, request, context): response = dj_pb.RawDataResponse() response.status.code = 0 data_source = self._fsm.get_data_source() if not self._validate_data_source_meta(request.data_source_meta, data_source.data_source_meta): response.status.code = -1 response.status.error_message = 'data source meta mismtach' return response if data_source.state != common_pb.DataSourceState.Processing: response.status.code = -2 response.status.error_message = ( "data source is not at processing state") return response rank_id = request.rank_id manifest = None finished = False manifest_manager = self._fsm.get_mainifest_manager() if request.HasField('sync_example_id'): partition_id = None if request.sync_example_id.partition_id >= 0: partition_id = request.sync_example_id.partition_id manifest, finished = manifest_manager.alloc_unallocated_partition( rank_id, partition_id) elif request.HasField('join_example'): partition_id = None if request.join_example.partition_id >= 0: partition_id = request.join_example.partition_id manifest, finished = manifest_manager.alloc_synced_partition( rank_id, partition_id) else: raise RuntimeError("unknown request type") if manifest is not None: response.manifest.MergeFrom(manifest) elif finished: response.finished.MergeFrom(empty_pb2.Empty()) return response