def _parse_cached_data(self, keys_value):
objector = self.serde()
if len(keys_value) != 1:
raise error.InvalidDataException("Invalid data: PObject should contain only 1 element" +
".\nmay be you should replace your 'reduce' or 'sum' method to 'aggregate'. \n" +
"See this page:\nhttp://bigflow.cloud/zh/rst/bigflow.transforms.html" +
"#bigflow.transforms.aggregate. \n The plain data is [" + str(keys_value) + "]")
kv = keys_value[0]
if len(kv.key) > 0:
raise error.InvalidDataException("Can not call get data in apply_values, \n" +
"SideInput is what you want, see this page:\n" +
"\thttp://bigflow.cloud/zh/guide.html#sideinputs")
return objector.deserialize(kv.value)
def _parse_cached_data(self, keys_value):
value_serde = self.serde()
key_serdes = self.key_serdes()
previous_keys = None
current_values = None
dict_root = dict()
is_pobject_value = isinstance(self.__inner_most_value(),
pobject.PObject)
for kv in keys_value:
result_value = value_serde.deserialize(kv.value)
if len(kv.key) == 0:
raise error.InvalidDataException("PTable should contain keys")
assert len(kv.key) == len(key_serdes), "key number is incorrect"
result_keys = map(lambda x: x[1].deserialize(x[0]),
zip(kv.key, key_serdes))
if is_pobject_value:
last_dict = PTable.__get_dict_from_keys(dict_root, result_keys)
last_key = result_keys[-1]
if last_key in last_dict:
raise error.InvalidDataException(
"Duplicate (keys, values) pair")
last_dict[last_key] = result_value
else:
if previous_keys is None or previous_keys != result_keys:
last_dict = PTable.__get_dict_from_keys(
dict_root, result_keys)
last_key = result_keys[-1]
if last_key in last_dict:
raise error.InvalidDataException(
"Duplicate (keys, values) pair!")
if is_pobject_value:
last_dict[last_key] = result_value
else:
current_values = []
last_dict[last_key] = current_values
previous_keys = result_keys
current_values.append(result_value)
return dict_root
def _parse_cached_data(self, keys_value):
objector = self.serde()
result = []
for kv in keys_value:
if len(kv.key) > 0:
raise error.InvalidDataException(
"Can not call get data in apply_values, \n" +
"SideInput is what you want, see this page:\n" +
"\thttp://bigflow.cloud/zh/guide.html#sideinputs")
result.append(objector.deserialize(kv.value))
return result