def test_deserialize_none(self):
fork_inst = jsons.fork()
jsons.set_deserializer(lambda *_, **__: 42,
cls=None,
fork_inst=fork_inst)
self.assertEqual(
42, jsons.load('Anything', cls=NoneType, fork_inst=fork_inst))
def get_serializer(cls):
if not LMTrainingConfig._serializer:
cls._serializer = jsons.fork()
jsons.set_deserializer(jsons.default_object_deserializer,
cls=cls,
fork_inst=cls._serializer)
jsons.set_deserializer(field_based_deserializer_func,
cls=Arch,
fork_inst=cls._serializer)
jsons.set_deserializer(field_based_deserializer_func,
cls=TrainingSchedule,
fork_inst=cls._serializer)
jsons.set_deserializer(field_based_deserializer_func,
cls=Optimizer,
fork_inst=cls._serializer)
jsons.set_deserializer(PrepFunction.deserializer,
cls=PrepFunction,
fork_inst=cls._serializer)
jsons.set_serializer(jsons.default_object_serializer,
cls=cls,
fork_inst=cls._serializer)
jsons.set_serializer(PrepFunction.serializer,
cls=PrepFunction,
fork_inst=cls._serializer)
return cls._serializer
def test_dict_hashkey_with_serializer(self):
def foo_serializer(obj, **kwargs):
return "{},{},{}".format(obj.a, obj.b, obj.c)
def foo_deserializer(obj, cls, **kwargs):
res = obj.split(',')
return Foo(a=int(res[0]), b=int(res[1]), c=int(res[2]))
jsons.set_serializer(foo_serializer, Foo, True)
jsons.set_deserializer(foo_deserializer, Foo, True)
bar = {Foo(1, 2, 3): D(a=42, b=39)}
dumped = jsons.dump(bar,
cls=Dict[Foo, D],
strict=True,
strip_privates=True,
strip_properties=True,
use_enum_name=True)
self.assertEqual(dumped, {'1,2,3': {"a": 42, "b": 39}})
loaded = jsons.load(dumped,
cls=Dict[Foo, D],
strict=True,
strip_privates=True,
strip_properties=True,
use_enum_name=True)
self.assertEqual(loaded, bar)
def test_set_custom_functions(self):
jsons.set_serializer(lambda *_, **__: 'custom_serializer', str)
jsons.set_deserializer(lambda *_, **__: 'custom_deserializer', str)
dumped = jsons.dump('serialize me')
loaded = jsons.load(dumped)
self.assertEqual(dumped, 'custom_serializer')
self.assertEqual(loaded, 'custom_deserializer')
def get_serializer(cls):
if not cls._serializer:
cls._serializer = jsons.fork()
jsons.set_deserializer(field_based_deserializer_func,
cls=Optimizer,
fork_inst=cls._serializer)
jsons.set_deserializer(field_based_deserializer_func,
cls=TrainingSchedule,
fork_inst=cls._serializer)
return cls._serializer
def __init__(self, dbname, qlog=None):
self.configLogger(qlog)
atlas = 'mongodb+srv://opus:[email protected]/asvp2'
local = 'mongodb://localhost:27017'
try :
self.client = MongoClient(local)
self.db = self.client[dbname]
for col in self.col_id:
lcol = self.db.get_collection(col)
name= lcol.name
except Exception as e :
logger.debug('Mongolink failed to {}'.format(e.__repr__()))
return
dispatcher.connect(self.entityChanged, signal=SIGNALS.ENTITYCHANGE, sender=dispatcher.Any)
# collections = self.db.collection_names()
# Search fo Entities and register them
m = __import__( 'Entities' )
for name, obj in inspect.getmembers(m):
if inspect.ismodule(obj) and getattr(obj, 'Entity'):
m1 = getattr(obj, name)
if inspect.isclass(m1) and m1.__name__ is not 'Entity':
self.entities[name] = m1
self.serializers[m1] = (jsons.get_serializer(m1))
self.deserializers[m1] = (jsons.get_deserializer(m1))
jsons.set_serializer(MongoLink.entity_serializer, m1)
jsons.set_deserializer(MongoLink.entity_deserializer, m1)
# Register new serializers to each entity class
# self.serializers[m1] = (jsons.get_serializer(BuildPressure))
# self.serializers[BuildPressure_pf] = (jsons.get_serializer(BuildPressure_pf))
#
# jsons.set_serializer(MongoLink.entity_serializer, BuildPressure)
# jsons.set_serializer(MongoLink.entity_serializer, BuildPressure_pf)
connected = True
logger.debug('MongoLink started...'.format())
def set_deserializers() -> None:
"""Set deserializers and validators for json to use to cast specific classes"""
# Local import to minimize jsons usage across files
from .group import ElementModP, ElementModQ, int_to_p_unchecked, int_to_q_unchecked
set_deserializer(lambda p_as_int, cls, **_: int_to_p_unchecked(p_as_int),
ElementModP)
set_validator(lambda p: p.is_in_bounds(), ElementModP)
set_deserializer(lambda q_as_int, cls, **_: int_to_q_unchecked(q_as_int),
ElementModQ)
set_validator(lambda q: q.is_in_bounds(), ElementModQ)
set_deserializer(
lambda none, cls, **_: None
if none == "None" else default_nonetype_deserializer(none),
NoneType,
)
set_deserializer(lambda dt, cls, **_: _deserialize_datetime(dt), datetime)
def get_serializer(cls):
if not cls._serializer:
cls._serializer = jsons.fork()
jsons.set_deserializer(jsons.default_object_deserializer,
cls=cls,
fork_inst=cls._serializer)
jsons.set_deserializer(field_based_deserializer_func,
cls=Arch,
fork_inst=cls._serializer)
jsons.set_deserializer(PrepFunction.deserializer,
cls=PrepFunction,
fork_inst=cls._serializer)
jsons.set_deserializer(Training.deserializer,
cls=Training,
fork_inst=cls._serializer)
jsons.set_serializer(jsons.default_object_serializer,
cls=cls,
fork_inst=cls._serializer)
jsons.set_serializer(PrepFunction.serializer,
cls=PrepFunction,
fork_inst=cls._serializer)
return cls._serializer
def set_deserializers() -> None:
"""Set deserializers and validators for json to use to cast specific classes"""
# Local import to minimize jsons usage across files
from .group import ElementModP, ElementModQ, int_to_p_unchecked, int_to_q_unchecked
from .proof import ProofUsage
set_deserializer(lambda p_as_int, cls, **_: int_to_p_unchecked(p_as_int),
ElementModP)
set_validator(lambda p: p.is_in_bounds(), ElementModP)
set_deserializer(lambda q_as_int, cls, **_: int_to_q_unchecked(q_as_int),
ElementModQ)
set_validator(lambda q: q.is_in_bounds(), ElementModQ)
set_deserializer(lambda usage_string, cls, **_: ProofUsage[usage_string],
ProofUsage)
def tearDownClass(cls):
jsons.set_serializer(default_primitive_serializer, str)
jsons.set_deserializer(default_string_deserializer, str)
for channel in ch_list:
add_listener(channel)
log.info(
f"Listener {subscriber_id} added to following queues: {ch_list}"
)
yield queue
finally:
log.info(f"Remove listener: {subscriber_id}")
for channel in ch_list:
remove_listener(channel)
self.active_listener.pop(subscriber_id, None)
await self.emit_event(CoreMessage.Disconnected, {
"subscriber_id": subscriber_id,
"channels": channels
})
async def emit_event(self, event_type: str, data: Json) -> None:
return await self.emit(Event(event_type, data))
async def emit(self, message: Message) -> None:
async def emit_by(name: str) -> None:
for listener in self.listeners.get(name, []):
await listener.put(message)
await emit_by(message.message_type) # inform specific listener
await emit_by("*") # inform "all" event listener
set_deserializer(Message.from_json, Message)
set_serializer(Message.message_to_json, Message)
def tearDown(cls):
jsons.set_serializer(jsons.default_primitive_serializer, Foo)
jsons.set_deserializer(jsons.default_string_deserializer, Foo)
# check if no property path is overlapping
def check_no_overlap() -> None:
existing_complex = [c for c in self.kinds.values() if isinstance(c, ComplexKind)]
update_complex = [c for c in to_update if isinstance(c, ComplexKind)]
ex = {p.path: p for k in existing_complex for p in k.resolved_properties()}
up = {p.path: p for k in update_complex for p in k.resolved_properties()}
def simple_kind_incompatible(p: PropertyPath) -> bool:
left = ex[p].kind
right = up[p].kind
return (left.fqn != right.fqn) and not (isinstance(left, AnyKind) or isinstance(right, AnyKind))
# Filter out duplicates that have the same kind or any side is any
non_unique = [a for a in ex.keys() & up.keys() if simple_kind_incompatible(a)]
if non_unique:
# PropertyPath -> str
name_by_kind = {p.path: k.fqn for k in update_complex for p in k.resolved_properties()}
message = ", ".join(f"{name_by_kind[a]}.{a} ({ex[a].kind.fqn} -> {up[a].kind.fqn})" for a in non_unique)
raise AttributeError(
f"Update not possible: following properties would be non unique having "
f"the same path but different type: {message}"
)
check_no_overlap()
return Model(updated)
# register serializer for this class
set_deserializer(Kind.from_json, Kind)
set_serializer(SimpleKind.to_json, SimpleKind)
**extra_args)
return Dataset(
name=dataset_name,
author=dct.get("author"),
description=dct.get("description", ""),
path_preprocessed=path_preprocessed,
path_dist=downloaded_data_dir / dct["path"],
dataloader=dataset_loader,
preprocessor=dct["preprocessor"],
)
else:
raise NotImplementedError()
jsons.set_deserializer(desearialize_dataset, Dataset)
jsons.set_serializer(Dataset.serealize, Dataset)
def get_all_linked_datasets(
datasets: Dict[str, Dataset]) -> Dict[str, Dataset]:
total = datasets.copy()
for dataset_name, dataset in datasets.items():
total.update({d.name: d for d in dataset.get_linked_datasets()})
return total
def train_and_test(
all_datasets: Dict[str, Dataset],
label_column: str) -> Tuple[Dict[str, Dataset], Dict[str, Dataset]]:
train, test = {}, {}
return first(relevant_ack, self.received_messages)
def pending_action_for(self, subscriber: Subscriber) -> Optional[Action]:
"""
In case this task is waiting for an action result from the given subscriber,
the relevant action is returned.
"""
state = self.current_state
if isinstance(state, PerformActionState):
message_type = state.perform.message_type
subscriptions = state.wait_for
if subscriber in subscriptions and self.ack_for(
message_type, subscriber) is None:
return Action(message_type, self.id, state.step.name)
return None
def begin_step(self) -> None:
log.info(f"Task {self.id}: begin step is: {self.current_step.name}")
# update the step started time, whenever a new state is entered
self.step_started_at = utc()
self.current_state.step_started()
set_deserializer(StepAction.from_json, StepAction, high_prio=False)
set_deserializer(Trigger.from_json, Trigger, high_prio=False)
set_deserializer(TaskCommand.from_json, TaskCommand, high_prio=False)
set_deserializer(Job.from_json, Job)
set_serializer(Job.to_json, Job)
set_deserializer(Workflow.from_json, Workflow)
set_serializer(Workflow.to_json, Workflow)
# pylint: disable=unused-argument
def env_var_deserializer(obj_dict, cls, **kwargs):
"""convert a dict to a subclass of AbstractEnvVarConfig"""
if obj_dict['source'] == EnvVarSource.SSM.value:
return SSMEnvVarConfig(obj_dict['path'])
if obj_dict['source'] == EnvVarSource.TEXT.value:
return TextEnvVarConfig(obj_dict['value'])
if obj_dict['source'] == EnvVarSource.UNSET.value:
return UnsetEnvVarConfig()
raise RuntimeError('Invalid Source')
jsons.set_deserializer(env_var_deserializer, AbstractEnvVarConfig)
# pylint: disable=too-many-arguments
class WrapperConfig:
def __init__(self,
configure_backend: bool = True,
pipeline_check: bool = True,
backend_check: bool = True,
plan_check: bool = True,
envvars: Dict[str, AbstractEnvVarConfig] = None,
backends: BackendsConfig = None,
depends_on: List[str] = None,
config: bool = True,
apply_automatically: bool = True,
plugins: Dict[str, str] = None):
pass
@abstractmethod
async def update_configs_model(self, kinds: List[Kind]) -> Model:
pass
@abstractmethod
def list_config_validation_ids(self) -> AsyncIterator[str]:
pass
@abstractmethod
async def get_config_validation(self,
cfg_id: str) -> Optional[ConfigValidation]:
pass
@abstractmethod
async def put_config_validation(
self, validation: ConfigValidation) -> ConfigValidation:
pass
@abstractmethod
async def config_yaml(self,
cfg_id: ConfigId,
revision: bool = False) -> Optional[str]:
pass
# register serializer for this class
set_deserializer(ConfigEntity.from_json, ConfigEntity)
set_serializer(ConfigEntity.to_json, ConfigEntity)
for task_name, task_json in dct["tasks"].items():
tasks[task_name] = jsons.load(
task_json,
Task,
task_name=task_name,
heuristic_path=path_config.heuristics,
datasets=datasets,
)
return BohrRepo(
dct["bohr_framework_version"],
tasks,
datasets,
linkers,
)
jsons.set_deserializer(deserialize_bohr_repo, BohrRepo)
jsons.set_serializer(BohrRepo.serealize, BohrRepo)
def load_bohr_repo(project_root: Optional[Path] = None) -> BohrRepo:
project_root = project_root or find_project_root()
bohr_repo = BohrRepo.load(project_root)
version_installed = version()
if str(bohr_repo.bohr_framework_version) != version_installed:
raise EnvironmentError(
f"Version of bohr framework from config: {bohr_repo.bohr_framework_version}. "
f"Version of bohr installed: {version_installed}")
return bohr_repo
f'{type(self).__name__} {self.config_version} '
f'to {type(self).__name__} {CONFIG_VERSION} object')
def create_comet_experiment(run_id: str):
experiment = Experiment()
experiment.set_name(run_id)
return experiment
@dataclass
class LMTrainingMetrics(object):
bin_entropy: Optional[float] = None
training_time_minutes_per_epoch: Optional[int] = None
n_epochs: Optional[int] = None
best_epoch: Optional[int] = None
trainable_params: Optional[int] = None
size_on_disk_mb: Optional[int] = None
config_version: str = CONFIG_VERSION
def __post_init__(self):
if self.config_version != CONFIG_VERSION:
raise TypeError(
f'Trying to deserealize '
f'{type(self).__name__} {self.config_version} '
f'to {type(self).__name__} {CONFIG_VERSION} object')
jsons.set_serializer(LMTrainingConfig.serializer, cls=LMTrainingConfig)
jsons.set_deserializer(LMTrainingConfig.deserializer, cls=LMTrainingConfig)
counters["sorts"] += 1
names["sort_names"].extend(sort.name for sort in part.sort)
if part.navigation:
navigation_analytics(part.navigation)
if part.with_clause:
with_clause_analytics(part.with_clause)
query_analytics(self)
return counters, names
@staticmethod
def mk_term(term: Union[str, Term], *args: Union[str, Term]) -> Term:
def make_term(t: Union[str, Term]) -> Term:
if isinstance(t, Term):
return t
elif isinstance(t, str):
return IsTerm([t])
else:
raise AttributeError(f"Expected term or string, but got {t}")
term_in = list(args)
term_in.insert(0, term)
terms = map(make_term, term_in)
# noinspection PyTypeChecker
return reduce(lambda l, r: CombinedTerm(l, "and", r), terms)
# register serializer for this class
set_deserializer(Term.from_json, Term)
import jsons import datetime from utils import utils # adding custom serializer / deserializer jsons.set_deserializer(utils.date_deserializer, str) jsons.set_serializer(utils.date_serializer, datetime.date) jsons.set_serializer(utils.datetime_serializer, datetime.datetime)
self.link = link
self.to = to
def __str__(self):
return f"{self.from_} -> {self.to}, linker: {self.link}"
def serealize(self, **kwargs) -> Dict[str, Any]:
dct = {"from": self.from_.name, "to": self.to.name}
if self.link:
dct["link"] = self.link.name
return dct
def desearialize_linker(
dct: Dict[str, Any],
cls,
datasets: Dict[str, Dataset],
data_dir: RelativePath,
**kwargs,
) -> "DatasetLinker":
extras = {}
if "link" in dct:
extras["link"] = datasets[dct["link"]]
return DatasetLinker(from_=datasets[dct["from"]],
to=datasets[dct["to"]],
**extras)
jsons.set_deserializer(desearialize_linker, DatasetLinker)
jsons.set_serializer(DatasetLinker.serealize, DatasetLinker)
# >>> jsons.loads('{"top_artifact": "artifacts.commit.Commit", "test_dataset_names": [], "train_dataset_names": []}', Task, project_root='/', task_name="x")
# """
test_datasets = {
dataset_name: datasets[dataset_name]
for dataset_name in dct["test_datasets"]
}
train_datasets = {
dataset_name: datasets[dataset_name]
for dataset_name in dct["train_datasets"]
}
try:
artifact = artifact_map[dct["top_artifact"]]
except KeyError:
artifact = load_class_by_full_path(dct["top_artifact"])
heuristic_groups = get_heuristic_module_list(artifact, heuristic_path)
return Task(
task_name,
dct["author"] if "author" in dct else None,
dct["description"] if "description" in dct else "",
artifact,
dct["label_categories"],
_train_datasets=train_datasets,
_test_datasets=test_datasets,
label_column_name=dct["label_column_name"],
heuristic_groups=heuristic_groups,
)
jsons.set_deserializer(deserialize_task, Task)
jsons.set_serializer(Task.serealize, Task)
import jsons # type: ignore
if TYPE_CHECKING: # pragma: no cover
from trakt.api import TraktApi
def any_deserializer(obj: Any, *args, **kwargs) -> Any:
return obj
def date_deserializer(obj: str, *args, **kwargs) -> Any:
return date.fromisoformat(obj)
jsons.set_deserializer(any_deserializer, Any)
jsons.set_deserializer(date_deserializer, date)
class AbstractBaseModel:
_client: "TraktApi"
@classmethod
def set_client(cls, client: "TraktApi") -> None:
cls._client = client
@property
def client(self) -> "TraktApi":
return self._client
def to_dict(self):
