def GetRawDataLatestTimeStamp(self, request, context):
response = dj_pb.RawDataResponse(timestamp=timestamp_pb2.Timestamp(
seconds=0))
meta_status = self._check_data_source_meta(request.data_source_meta)
if meta_status.code != 0:
response.status.MergeFrom(meta_status)
return response
manifest_manager = self._fsm.get_mainifest_manager()
ts = manifest_manager.get_raw_date_latest_timestamp(
request.partition_id)
if ts is not None:
response.timestamp.MergeFrom(ts)
return response
def RequestJoinPartition(self, request, context):
response = dj_pb.RawDataResponse()
meta_status = self._check_data_source_meta(request.data_source_meta)
if meta_status.code != 0:
response.status.MergeFrom(meta_status)
return response
rank_status = self._check_rank_id(request.rank_id)
if rank_status.code != 0:
response.status.MergeFrom(rank_status)
return response
data_source = self._fsm.get_data_source()
if data_source.state != common_pb.DataSourceState.Processing:
response.status.code = -3
response.status.error_message = \
"data source is not at processing state"
else:
manifest_manager = self._fsm.get_mainifest_manager()
rank_id = request.rank_id
manifest = None
partition_id = None if request.partition_id < 0 \
else request.partition_id
if request.HasField('sync_example_id'):
manifest = manifest_manager.alloc_sync_exampld_id(
rank_id, partition_id
)
elif request.HasField('join_example'):
manifest = manifest_manager.alloc_join_example(
rank_id, partition_id
)
else:
response.status.code = -4
response.status.error_message = "request not support"
if response.status.code == 0:
if manifest is not None:
response.manifest.MergeFrom(manifest)
else:
assert partition_id is None, \
"only the request without appoint partition "\
"support response no manifest"
response.finished.MergeFrom(empty_pb2.Empty())
return response
def RequestJoinPartition(self, request, context):
response = dj_pb.RawDataResponse()
response.status.code = 0
data_source = self._fsm.get_data_source()
if not self._validate_data_source_meta(request.data_source_meta,
data_source.data_source_meta):
response.status.code = -1
response.status.error_message = 'data source meta mismtach'
return response
if data_source.state != common_pb.DataSourceState.Processing:
response.status.code = -2
response.status.error_message = (
"data source is not at processing state")
return response
rank_id = request.rank_id
manifest = None
finished = False
manifest_manager = self._fsm.get_mainifest_manager()
if request.HasField('sync_example_id'):
partition_id = None
if request.sync_example_id.partition_id >= 0:
partition_id = request.sync_example_id.partition_id
manifest, finished = manifest_manager.alloc_unallocated_partition(
rank_id, partition_id)
elif request.HasField('join_example'):
partition_id = None
if request.join_example.partition_id >= 0:
partition_id = request.join_example.partition_id
manifest, finished = manifest_manager.alloc_synced_partition(
rank_id, partition_id)
else:
raise RuntimeError("unknown request type")
if manifest is not None:
response.manifest.MergeFrom(manifest)
elif finished:
response.finished.MergeFrom(empty_pb2.Empty())
return response