def test_compare_lazy_var_and_simple_var(self): def _load_dict(): return {str(x): str(x) for x in xrange(100)} my_lazy_var = lazy_var.declare(_load_dict) def _use_dict_by_lazy_var(): my_dict = my_lazy_var.get() values = '\t'.join(my_dict.values()) return values dump_object_before_call_get = cloudpickle.dumps(_use_dict_by_lazy_var) my_lazy_var.get() dump_object_after_call_get = cloudpickle.dumps(_use_dict_by_lazy_var) # assert equal self.assertEqual(dump_object_before_call_get, dump_object_after_call_get) my_simple_var = {} def _load_dict_to_simple_var(): for x in xrange(100): my_simple_var[str(x)] = str(x) return my_simple_var def _use_dict_by_simple_var(): values = '\t'.join(my_simple_var.values()) return values dump_object_before_load_dict = cloudpickle.dumps(_use_dict_by_simple_var) _load_dict_to_simple_var() dump_object_after_load_dict = cloudpickle.dumps(_use_dict_by_simple_var) # assert not equal self.assertNotEqual(dump_object_before_load_dict, dump_object_after_load_dict)
def __init__(self, user_input_base): super(_UserDefineFileBase, self).__init__('user_define_format') self.input_format = entity.Entity.of( entity.Entity.loader, cloudpickle.dumps(_LoaderImpl(user_input_base))) self.objector = user_input_base.get_serde() self._user_input_base = user_input_base
def to_proto_message(self): from flume.proto import entity_pb2 from bigflow.core import entity_names pb_entity = entity_pb2.PbEntity() pb_entity.name = entity_names.__dict__['PythonImplFunctor'] config = {} config['fn'] = self.__fn config['expect_iterable'] = self.__expect_iterable pb_entity.config = cloudpickle.dumps(config) return pb_entity
def get_entity_config(self): from bigflow_python.proto import processor_pb2 processor = processor_pb2.PbPythonProcessorConfig() if self.__config is not None: processor.config = cloudpickle.dumps(self.__config) for fn in self.__fns: fn = Functor.of(fn) processor.functor.add().CopyFrom(fn.to_proto_message()) for side_input in self.__side_inputs: side_input_type = processor_pb2.POBJECT_TYPE if isinstance(side_input, pcollection.PCollection): side_input_type = processor_pb2.PCOLLECTION_TYPE processor.side_input_type.append(side_input_type) return processor.SerializeToString()
def _generate_resource_message(self): folder = entity.ENTITY_FOLDER if os.path.exists(folder): for file_name in os.listdir(folder): src_file = os.path.join(folder, file_name) target_file = os.path.join(entity.FLUME_WORKER_ENTITY_FOLDER, file_name) self.add_file(src_file, target_file) def _sorted_hooks(hooks_dict): result = [] for name in sorted(hooks_dict.keys()): result.append(hooks_dict[name]) return result import copy resource = copy.deepcopy(self._resource) from bigflow.core.serde import cloudpickle resource.add_file_from_bytes( cloudpickle.dumps(_sorted_hooks(self._init_hooks)), ".init_hooks") resource.add_file_from_bytes( cloudpickle.dumps(_sorted_hooks(self._fini_hooks)), ".fini_hooks") self._resource_message = resource.to_proto_message()
def test_compare_lazy_var_and_simple_var(self): def _load_dict(): return {str(x): str(x) for x in xrange(100)} my_lazy_var = lazy_var.declare(_load_dict) def _use_dict_by_lazy_var(): my_dict = my_lazy_var.get() values = '\t'.join(my_dict.values()) return values dump_object_before_call_get = cloudpickle.dumps(_use_dict_by_lazy_var) my_lazy_var.get() dump_object_after_call_get = cloudpickle.dumps(_use_dict_by_lazy_var) # assert equal self.assertEqual(dump_object_before_call_get, dump_object_after_call_get) my_simple_var = {} def _load_dict_to_simple_var(): for x in xrange(100): my_simple_var[str(x)] = str(x) return my_simple_var def _use_dict_by_simple_var(): values = '\t'.join(my_simple_var.values()) return values dump_object_before_load_dict = cloudpickle.dumps( _use_dict_by_simple_var) _load_dict_to_simple_var() dump_object_after_load_dict = cloudpickle.dumps( _use_dict_by_simple_var) # assert not equal self.assertNotEqual(dump_object_before_load_dict, dump_object_after_load_dict)
def __init__(self, name="", operator=None, message=None): if message is None: if len(name) == 0: raise error.InvalidLogicalPlanException("Invalid name for entity.") if operator is None: raise error.InvalidLogicalPlanException("Invalid operator(None) for entity.") if isinstance(operator, EntitiedBySelf): self.__name = operator.get_entity_name() self.__config = operator.get_entity_config() elif isinstance(operator, str): self.__name = name self.__config = operator else: self.__name = name self.__config = cloudpickle.dumps(operator) else: self.from_proto_message(message)
def __init__(self, name="", operator=None, message=None): if message is None: if len(name) == 0: raise error.InvalidLogicalPlanException( "Invalid name for entity.") if operator is None: raise error.InvalidLogicalPlanException( "Invalid operator(None) for entity.") if isinstance(operator, EntitiedBySelf): self.__name = operator.get_entity_name() self.__config = operator.get_entity_config() elif isinstance(operator, str): self.__name = name self.__config = operator else: self.__name = name self.__config = cloudpickle.dumps(operator) else: self.from_proto_message(message)
def get_entity_config(self): """ get_entity_config """ return cloudpickle.dumps(self)
def get_entity_config(self): """ 内部函数 """ return cloudpickle.dumps(self)
def get_entity_config(self): return cloudpickle.dumps(self)
def config(self): """Pass serializers to cpp runtime""" return cloudpickle.dumps(self.serializers)
def config(self): """Config: Pass serialized arguments to cpp runtime""" return cloudpickle.dumps((self._is_serialize, self._objector, self._apply_index))
def config(self): """ Config: Pass sep, fields_type arguments to cpp runtime""" return cloudpickle.dumps((self._sep, self._fields_type, self._ignore_overflow, self._ignore_illegal_line))
def config(self): """Config: Pass serialized arguments to cpp runtime""" return cloudpickle.dumps( (self._is_serialize, self._objector, self._apply_index))
def config(self): """ Config: Pass sep, fields_type arguments to cpp runtime""" return cloudpickle.dumps( (self._sep, self._fields_type, self._ignore_overflow, self._ignore_illegal_line))