def test_deserialize_none(self): fork_inst = jsons.fork() jsons.set_deserializer(lambda *_, **__: 42, cls=None, fork_inst=fork_inst) self.assertEqual( 42, jsons.load('Anything', cls=NoneType, fork_inst=fork_inst))
def get_serializer(cls): if not LMTrainingConfig._serializer: cls._serializer = jsons.fork() jsons.set_deserializer(jsons.default_object_deserializer, cls=cls, fork_inst=cls._serializer) jsons.set_deserializer(field_based_deserializer_func, cls=Arch, fork_inst=cls._serializer) jsons.set_deserializer(field_based_deserializer_func, cls=TrainingSchedule, fork_inst=cls._serializer) jsons.set_deserializer(field_based_deserializer_func, cls=Optimizer, fork_inst=cls._serializer) jsons.set_deserializer(PrepFunction.deserializer, cls=PrepFunction, fork_inst=cls._serializer) jsons.set_serializer(jsons.default_object_serializer, cls=cls, fork_inst=cls._serializer) jsons.set_serializer(PrepFunction.serializer, cls=PrepFunction, fork_inst=cls._serializer) return cls._serializer
def test_dict_hashkey_with_serializer(self): def foo_serializer(obj, **kwargs): return "{},{},{}".format(obj.a, obj.b, obj.c) def foo_deserializer(obj, cls, **kwargs): res = obj.split(',') return Foo(a=int(res[0]), b=int(res[1]), c=int(res[2])) jsons.set_serializer(foo_serializer, Foo, True) jsons.set_deserializer(foo_deserializer, Foo, True) bar = {Foo(1, 2, 3): D(a=42, b=39)} dumped = jsons.dump(bar, cls=Dict[Foo, D], strict=True, strip_privates=True, strip_properties=True, use_enum_name=True) self.assertEqual(dumped, {'1,2,3': {"a": 42, "b": 39}}) loaded = jsons.load(dumped, cls=Dict[Foo, D], strict=True, strip_privates=True, strip_properties=True, use_enum_name=True) self.assertEqual(loaded, bar)
def test_set_custom_functions(self): jsons.set_serializer(lambda *_, **__: 'custom_serializer', str) jsons.set_deserializer(lambda *_, **__: 'custom_deserializer', str) dumped = jsons.dump('serialize me') loaded = jsons.load(dumped) self.assertEqual(dumped, 'custom_serializer') self.assertEqual(loaded, 'custom_deserializer')
def get_serializer(cls): if not cls._serializer: cls._serializer = jsons.fork() jsons.set_deserializer(field_based_deserializer_func, cls=Optimizer, fork_inst=cls._serializer) jsons.set_deserializer(field_based_deserializer_func, cls=TrainingSchedule, fork_inst=cls._serializer) return cls._serializer
def __init__(self, dbname, qlog=None): self.configLogger(qlog) atlas = 'mongodb+srv://opus:[email protected]/asvp2' local = 'mongodb://localhost:27017' try : self.client = MongoClient(local) self.db = self.client[dbname] for col in self.col_id: lcol = self.db.get_collection(col) name= lcol.name except Exception as e : logger.debug('Mongolink failed to {}'.format(e.__repr__())) return dispatcher.connect(self.entityChanged, signal=SIGNALS.ENTITYCHANGE, sender=dispatcher.Any) # collections = self.db.collection_names() # Search fo Entities and register them m = __import__( 'Entities' ) for name, obj in inspect.getmembers(m): if inspect.ismodule(obj) and getattr(obj, 'Entity'): m1 = getattr(obj, name) if inspect.isclass(m1) and m1.__name__ is not 'Entity': self.entities[name] = m1 self.serializers[m1] = (jsons.get_serializer(m1)) self.deserializers[m1] = (jsons.get_deserializer(m1)) jsons.set_serializer(MongoLink.entity_serializer, m1) jsons.set_deserializer(MongoLink.entity_deserializer, m1) # Register new serializers to each entity class # self.serializers[m1] = (jsons.get_serializer(BuildPressure)) # self.serializers[BuildPressure_pf] = (jsons.get_serializer(BuildPressure_pf)) # # jsons.set_serializer(MongoLink.entity_serializer, BuildPressure) # jsons.set_serializer(MongoLink.entity_serializer, BuildPressure_pf) connected = True logger.debug('MongoLink started...'.format())
def set_deserializers() -> None: """Set deserializers and validators for json to use to cast specific classes""" # Local import to minimize jsons usage across files from .group import ElementModP, ElementModQ, int_to_p_unchecked, int_to_q_unchecked set_deserializer(lambda p_as_int, cls, **_: int_to_p_unchecked(p_as_int), ElementModP) set_validator(lambda p: p.is_in_bounds(), ElementModP) set_deserializer(lambda q_as_int, cls, **_: int_to_q_unchecked(q_as_int), ElementModQ) set_validator(lambda q: q.is_in_bounds(), ElementModQ) set_deserializer( lambda none, cls, **_: None if none == "None" else default_nonetype_deserializer(none), NoneType, ) set_deserializer(lambda dt, cls, **_: _deserialize_datetime(dt), datetime)
def get_serializer(cls): if not cls._serializer: cls._serializer = jsons.fork() jsons.set_deserializer(jsons.default_object_deserializer, cls=cls, fork_inst=cls._serializer) jsons.set_deserializer(field_based_deserializer_func, cls=Arch, fork_inst=cls._serializer) jsons.set_deserializer(PrepFunction.deserializer, cls=PrepFunction, fork_inst=cls._serializer) jsons.set_deserializer(Training.deserializer, cls=Training, fork_inst=cls._serializer) jsons.set_serializer(jsons.default_object_serializer, cls=cls, fork_inst=cls._serializer) jsons.set_serializer(PrepFunction.serializer, cls=PrepFunction, fork_inst=cls._serializer) return cls._serializer
def set_deserializers() -> None: """Set deserializers and validators for json to use to cast specific classes""" # Local import to minimize jsons usage across files from .group import ElementModP, ElementModQ, int_to_p_unchecked, int_to_q_unchecked from .proof import ProofUsage set_deserializer(lambda p_as_int, cls, **_: int_to_p_unchecked(p_as_int), ElementModP) set_validator(lambda p: p.is_in_bounds(), ElementModP) set_deserializer(lambda q_as_int, cls, **_: int_to_q_unchecked(q_as_int), ElementModQ) set_validator(lambda q: q.is_in_bounds(), ElementModQ) set_deserializer(lambda usage_string, cls, **_: ProofUsage[usage_string], ProofUsage)
def tearDownClass(cls): jsons.set_serializer(default_primitive_serializer, str) jsons.set_deserializer(default_string_deserializer, str)
for channel in ch_list: add_listener(channel) log.info( f"Listener {subscriber_id} added to following queues: {ch_list}" ) yield queue finally: log.info(f"Remove listener: {subscriber_id}") for channel in ch_list: remove_listener(channel) self.active_listener.pop(subscriber_id, None) await self.emit_event(CoreMessage.Disconnected, { "subscriber_id": subscriber_id, "channels": channels }) async def emit_event(self, event_type: str, data: Json) -> None: return await self.emit(Event(event_type, data)) async def emit(self, message: Message) -> None: async def emit_by(name: str) -> None: for listener in self.listeners.get(name, []): await listener.put(message) await emit_by(message.message_type) # inform specific listener await emit_by("*") # inform "all" event listener set_deserializer(Message.from_json, Message) set_serializer(Message.message_to_json, Message)
def tearDown(cls): jsons.set_serializer(jsons.default_primitive_serializer, Foo) jsons.set_deserializer(jsons.default_string_deserializer, Foo)
# check if no property path is overlapping def check_no_overlap() -> None: existing_complex = [c for c in self.kinds.values() if isinstance(c, ComplexKind)] update_complex = [c for c in to_update if isinstance(c, ComplexKind)] ex = {p.path: p for k in existing_complex for p in k.resolved_properties()} up = {p.path: p for k in update_complex for p in k.resolved_properties()} def simple_kind_incompatible(p: PropertyPath) -> bool: left = ex[p].kind right = up[p].kind return (left.fqn != right.fqn) and not (isinstance(left, AnyKind) or isinstance(right, AnyKind)) # Filter out duplicates that have the same kind or any side is any non_unique = [a for a in ex.keys() & up.keys() if simple_kind_incompatible(a)] if non_unique: # PropertyPath -> str name_by_kind = {p.path: k.fqn for k in update_complex for p in k.resolved_properties()} message = ", ".join(f"{name_by_kind[a]}.{a} ({ex[a].kind.fqn} -> {up[a].kind.fqn})" for a in non_unique) raise AttributeError( f"Update not possible: following properties would be non unique having " f"the same path but different type: {message}" ) check_no_overlap() return Model(updated) # register serializer for this class set_deserializer(Kind.from_json, Kind) set_serializer(SimpleKind.to_json, SimpleKind)
**extra_args) return Dataset( name=dataset_name, author=dct.get("author"), description=dct.get("description", ""), path_preprocessed=path_preprocessed, path_dist=downloaded_data_dir / dct["path"], dataloader=dataset_loader, preprocessor=dct["preprocessor"], ) else: raise NotImplementedError() jsons.set_deserializer(desearialize_dataset, Dataset) jsons.set_serializer(Dataset.serealize, Dataset) def get_all_linked_datasets( datasets: Dict[str, Dataset]) -> Dict[str, Dataset]: total = datasets.copy() for dataset_name, dataset in datasets.items(): total.update({d.name: d for d in dataset.get_linked_datasets()}) return total def train_and_test( all_datasets: Dict[str, Dataset], label_column: str) -> Tuple[Dict[str, Dataset], Dict[str, Dataset]]: train, test = {}, {}
return first(relevant_ack, self.received_messages) def pending_action_for(self, subscriber: Subscriber) -> Optional[Action]: """ In case this task is waiting for an action result from the given subscriber, the relevant action is returned. """ state = self.current_state if isinstance(state, PerformActionState): message_type = state.perform.message_type subscriptions = state.wait_for if subscriber in subscriptions and self.ack_for( message_type, subscriber) is None: return Action(message_type, self.id, state.step.name) return None def begin_step(self) -> None: log.info(f"Task {self.id}: begin step is: {self.current_step.name}") # update the step started time, whenever a new state is entered self.step_started_at = utc() self.current_state.step_started() set_deserializer(StepAction.from_json, StepAction, high_prio=False) set_deserializer(Trigger.from_json, Trigger, high_prio=False) set_deserializer(TaskCommand.from_json, TaskCommand, high_prio=False) set_deserializer(Job.from_json, Job) set_serializer(Job.to_json, Job) set_deserializer(Workflow.from_json, Workflow) set_serializer(Workflow.to_json, Workflow)
# pylint: disable=unused-argument def env_var_deserializer(obj_dict, cls, **kwargs): """convert a dict to a subclass of AbstractEnvVarConfig""" if obj_dict['source'] == EnvVarSource.SSM.value: return SSMEnvVarConfig(obj_dict['path']) if obj_dict['source'] == EnvVarSource.TEXT.value: return TextEnvVarConfig(obj_dict['value']) if obj_dict['source'] == EnvVarSource.UNSET.value: return UnsetEnvVarConfig() raise RuntimeError('Invalid Source') jsons.set_deserializer(env_var_deserializer, AbstractEnvVarConfig) # pylint: disable=too-many-arguments class WrapperConfig: def __init__(self, configure_backend: bool = True, pipeline_check: bool = True, backend_check: bool = True, plan_check: bool = True, envvars: Dict[str, AbstractEnvVarConfig] = None, backends: BackendsConfig = None, depends_on: List[str] = None, config: bool = True, apply_automatically: bool = True, plugins: Dict[str, str] = None):
pass @abstractmethod async def update_configs_model(self, kinds: List[Kind]) -> Model: pass @abstractmethod def list_config_validation_ids(self) -> AsyncIterator[str]: pass @abstractmethod async def get_config_validation(self, cfg_id: str) -> Optional[ConfigValidation]: pass @abstractmethod async def put_config_validation( self, validation: ConfigValidation) -> ConfigValidation: pass @abstractmethod async def config_yaml(self, cfg_id: ConfigId, revision: bool = False) -> Optional[str]: pass # register serializer for this class set_deserializer(ConfigEntity.from_json, ConfigEntity) set_serializer(ConfigEntity.to_json, ConfigEntity)
for task_name, task_json in dct["tasks"].items(): tasks[task_name] = jsons.load( task_json, Task, task_name=task_name, heuristic_path=path_config.heuristics, datasets=datasets, ) return BohrRepo( dct["bohr_framework_version"], tasks, datasets, linkers, ) jsons.set_deserializer(deserialize_bohr_repo, BohrRepo) jsons.set_serializer(BohrRepo.serealize, BohrRepo) def load_bohr_repo(project_root: Optional[Path] = None) -> BohrRepo: project_root = project_root or find_project_root() bohr_repo = BohrRepo.load(project_root) version_installed = version() if str(bohr_repo.bohr_framework_version) != version_installed: raise EnvironmentError( f"Version of bohr framework from config: {bohr_repo.bohr_framework_version}. " f"Version of bohr installed: {version_installed}") return bohr_repo
f'{type(self).__name__} {self.config_version} ' f'to {type(self).__name__} {CONFIG_VERSION} object') def create_comet_experiment(run_id: str): experiment = Experiment() experiment.set_name(run_id) return experiment @dataclass class LMTrainingMetrics(object): bin_entropy: Optional[float] = None training_time_minutes_per_epoch: Optional[int] = None n_epochs: Optional[int] = None best_epoch: Optional[int] = None trainable_params: Optional[int] = None size_on_disk_mb: Optional[int] = None config_version: str = CONFIG_VERSION def __post_init__(self): if self.config_version != CONFIG_VERSION: raise TypeError( f'Trying to deserealize ' f'{type(self).__name__} {self.config_version} ' f'to {type(self).__name__} {CONFIG_VERSION} object') jsons.set_serializer(LMTrainingConfig.serializer, cls=LMTrainingConfig) jsons.set_deserializer(LMTrainingConfig.deserializer, cls=LMTrainingConfig)
counters["sorts"] += 1 names["sort_names"].extend(sort.name for sort in part.sort) if part.navigation: navigation_analytics(part.navigation) if part.with_clause: with_clause_analytics(part.with_clause) query_analytics(self) return counters, names @staticmethod def mk_term(term: Union[str, Term], *args: Union[str, Term]) -> Term: def make_term(t: Union[str, Term]) -> Term: if isinstance(t, Term): return t elif isinstance(t, str): return IsTerm([t]) else: raise AttributeError(f"Expected term or string, but got {t}") term_in = list(args) term_in.insert(0, term) terms = map(make_term, term_in) # noinspection PyTypeChecker return reduce(lambda l, r: CombinedTerm(l, "and", r), terms) # register serializer for this class set_deserializer(Term.from_json, Term)
import jsons import datetime from utils import utils # adding custom serializer / deserializer jsons.set_deserializer(utils.date_deserializer, str) jsons.set_serializer(utils.date_serializer, datetime.date) jsons.set_serializer(utils.datetime_serializer, datetime.datetime)
self.link = link self.to = to def __str__(self): return f"{self.from_} -> {self.to}, linker: {self.link}" def serealize(self, **kwargs) -> Dict[str, Any]: dct = {"from": self.from_.name, "to": self.to.name} if self.link: dct["link"] = self.link.name return dct def desearialize_linker( dct: Dict[str, Any], cls, datasets: Dict[str, Dataset], data_dir: RelativePath, **kwargs, ) -> "DatasetLinker": extras = {} if "link" in dct: extras["link"] = datasets[dct["link"]] return DatasetLinker(from_=datasets[dct["from"]], to=datasets[dct["to"]], **extras) jsons.set_deserializer(desearialize_linker, DatasetLinker) jsons.set_serializer(DatasetLinker.serealize, DatasetLinker)
# >>> jsons.loads('{"top_artifact": "artifacts.commit.Commit", "test_dataset_names": [], "train_dataset_names": []}', Task, project_root='/', task_name="x") # """ test_datasets = { dataset_name: datasets[dataset_name] for dataset_name in dct["test_datasets"] } train_datasets = { dataset_name: datasets[dataset_name] for dataset_name in dct["train_datasets"] } try: artifact = artifact_map[dct["top_artifact"]] except KeyError: artifact = load_class_by_full_path(dct["top_artifact"]) heuristic_groups = get_heuristic_module_list(artifact, heuristic_path) return Task( task_name, dct["author"] if "author" in dct else None, dct["description"] if "description" in dct else "", artifact, dct["label_categories"], _train_datasets=train_datasets, _test_datasets=test_datasets, label_column_name=dct["label_column_name"], heuristic_groups=heuristic_groups, ) jsons.set_deserializer(deserialize_task, Task) jsons.set_serializer(Task.serealize, Task)
import jsons # type: ignore if TYPE_CHECKING: # pragma: no cover from trakt.api import TraktApi def any_deserializer(obj: Any, *args, **kwargs) -> Any: return obj def date_deserializer(obj: str, *args, **kwargs) -> Any: return date.fromisoformat(obj) jsons.set_deserializer(any_deserializer, Any) jsons.set_deserializer(date_deserializer, date) class AbstractBaseModel: _client: "TraktApi" @classmethod def set_client(cls, client: "TraktApi") -> None: cls._client = client @property def client(self) -> "TraktApi": return self._client def to_dict(self):