def __init__(self, hparams: om.DictConfig):
super().__init__()
if not isinstance(hparams, om.DictConfig):
hparams = om.DictConfig(hparams)
self.hparams = om.OmegaConf.to_container(hparams, resolve=True)
# Instantiate datasets (Hydra compat)
self.dataset = hu.instantiate(hparams.dataset)
# Instantiate network modules
self.gen = hu.instantiate(hparams.gen)
self.dis = hu.instantiate(hparams.dis)
self.type = hparams.var.type
if hparams.var.ema:
self.gen_ema = hu.instantiate(hparams.gen)
# Instantiate optimizers & schedulers
self.gen_optim = hu.instantiate(hparams.gen_opt, self.gen.parameters())
self.dis_optim = hu.instantiate(hparams.dis_opt, self.dis.parameters())
# Instantiate losses
self.lambda_recon = hparams.var.lambda_recon
self.lambda_gp = hparams.var.lambda_gp
self.recon_loss = hu.instantiate(hparams.recon_loss, self.dis)
self.adv_loss = hu.instantiate(hparams.adv_loss)
def template_config():
test_template = {
"project_name": "test_project",
"default_branch": "master",
"owner": "owner_test",
"author": "owner_test",
"email": "*****@*****.**",
"copyright_year": 2021,
"copyright_holder": "owner_test",
"project_url": "https://github.com/owner_test/test_project",
"bot_name": "owner_test",
"bot_email": "*****@*****.**",
"license": "MIT",
"description": "test description of the project",
"python_versions": ["3.6", "3.7", "3.8", "3.9"],
"docker_image": "fragiletech/ubuntu18.04-cuda-11.0-py39",
"ci_python_version": "3.8",
"ci_ubuntu_version": "ubuntu-20.04",
"ci_extra": "",
"docstring_checks": False,
"pyproject_extra": "",
"git_message": "Test init commit",
}
return omegaconf.DictConfig(test_template)
def _cell(*args, **kwargs):
autoname = kwargs.pop("autoname", True)
name = kwargs.pop("name", None)
cache = kwargs.pop("cache", True)
info = kwargs.pop("info", omegaconf.DictConfig({}))
prefix = kwargs.pop("prefix", func.__name__)
max_name_length = kwargs.pop("max_name_length", MAX_NAME_LENGTH)
sig = inspect.signature(func)
args_as_kwargs = dict(zip(sig.parameters.keys(), args))
args_as_kwargs.update(**copy.deepcopy(kwargs))
default = {
p.name: p.default
for p in sig.parameters.values()
if not p.default == inspect._empty
}
changed = args_as_kwargs
default = default
full = copy.deepcopy(default)
full.update(**args_as_kwargs)
named_args_list = [
f"{key}={clean_value(changed[key])}" for key in sorted(changed.keys())
]
named_args_string = "_".join(named_args_list)
named_args_hash = hashlib.md5(named_args_string.encode()).hexdigest()[:8]
name_signature = (
clean_name(f"{prefix}_{named_args_hash}") if named_args_list else prefix
)
name = name or name_signature
decorator = kwargs.pop("decorator", None)
name = get_name_short(name, max_name_length=max_name_length)
if (
"args" not in sig.parameters
and "kwargs" not in sig.parameters
and "settings" not in sig.parameters
):
for key in kwargs.keys():
if key not in sig.parameters.keys():
raise TypeError(
f"{func.__name__}() got invalid argument `{key}`\n"
f"valid arguments are {list(sig.parameters.keys())}"
)
if cache and name in CACHE:
# print(f"CACHE LOAD {name} {func.__name__}({arguments})")
return CACHE[name]
else:
# print(f"BUILD {name} {func.__name__}({arguments})")
assert callable(
func
), f"{func} got decorated with @cell! @cell decorator is only for functions"
component = func(*args, **kwargs)
if not isinstance(component, Component):
raise CellReturnTypeError(
f"function `{func.__name__}` return type = `{type(component)}`",
"make sure that functions with @cell decorator return a Component",
)
if (
component.get_child_name
and hasattr(component, "info_child")
and hasattr(component.info_child, "name")
):
component_name = f"{component.info_child.name}_{name}"
component_name = get_name_short(
component_name, max_name_length=max_name_length
)
else:
component_name = name
if autoname:
component.name = component_name
component.info.name = component_name
component.info.module = func.__module__
component.info.function_name = func.__name__
component.info.info_version = INFO_VERSION
component.info.changed = clean_dict(changed)
component.info.default = clean_dict(default)
component.info.full = clean_dict(full)
component.info.update(**info)
if decorator:
if not callable(decorator):
raise ValueError(
f"decorator = {type(decorator)} needs to be callable"
)
component_new = decorator(component)
component = component_new or component
component.lock()
CACHE[name] = component
# avoid_duplicated_cells
if name in CACHE_IMPORTED_CELLS:
c = CACHE_IMPORTED_CELLS.pop(name)
i = 1
new_name = f"{c.name}${i}"
while new_name in CACHE or new_name in CACHE_IMPORTED_CELLS:
i += 1
new_name = f"{c.name}${i}"
c.name = new_name
CACHE_IMPORTED_CELLS[new_name] = c
return component
import torch
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint
from pytorch_lightning.loggers.neptune import NeptuneLogger
from tqdm import tqdm
from model import LyftMultiModel, set_seed
# from logzero import logger
set_seed(42)
# Hyperparameters
cfg = load_config_data(
"/data/lyft-motion-prediction-autonomous-vehicles/lyft-config-files/agent_motion_config.yaml"
)
cfg = omegaconf.DictConfig(cfg)
name_for_save = 'Big_training'
epochs = cfg["model_params"]["epochs"]
learning_rate = cfg["model_params"]["lr"]
training_percentage = cfg["train_data_loader"]["training_percentage"]
validation_percentage = cfg["val_data_loader"]["validation_percentage"]
API_KEY = os.environ.get('NEPTUNE_API_KEY')
neptune_logger = NeptuneLogger(
api_key=API_KEY,
project_name='hvergnes/KaggleResNet',
params={
'epoch_nr': epochs,
'learning_rate': learning_rate,
'train_size': training_percentage,
'test_size': validation_percentage
def _cell(*args, **kwargs):
prefix = kwargs.pop("prefix", func.__name__)
autoname = kwargs.pop("autoname", True)
cache = kwargs.pop("cache", True)
info = kwargs.pop("info", omegaconf.DictConfig({}))
sig = inspect.signature(func)
args_as_kwargs = dict(zip(sig.parameters.keys(), args))
args_as_kwargs.update(**kwargs)
args_as_kwargs_string_list = [
f"{key}={clean_value(args_as_kwargs[key])}"
for key in sorted(args_as_kwargs.keys())
]
arguments = "_".join(args_as_kwargs_string_list)
arguments_hash = hashlib.md5(arguments.encode()).hexdigest()[:8]
name_long = name = (
clean_name(f"{prefix}_{arguments_hash}") if arguments else prefix
)
decorator = kwargs.pop("decorator", None)
if len(name) > MAX_NAME_LENGTH:
name_hash = hashlib.md5(name.encode()).hexdigest()[:8]
name = f"{name[:(MAX_NAME_LENGTH - 9)]}_{name_hash}"
name_component = kwargs.pop("name", name)
if (
"args" not in sig.parameters
and "kwargs" not in sig.parameters
and "settings" not in sig.parameters
):
for key in kwargs.keys():
if key not in sig.parameters.keys():
raise TypeError(
f"{func.__name__}() got invalid argument `{key}`\n"
f"valid arguments are {list(sig.parameters.keys())}"
)
if cache and name in CACHE:
# print(f"CACHE {func.__name__}({kwargs_repr})")
return CACHE[name]
else:
# print(f"BUILD {name} {func.__name__}({arguments})")
# print(f"BUILD {name}, {name_long}")
assert callable(
func
), f"{func} got decorated with @cell! @cell decorator is only for functions"
component = func(*args, **kwargs)
if decorator:
assert callable(
decorator
), f"decorator = {type(decorator)} needs to be callable"
decorator(component)
if not isinstance(component, Component):
raise CellReturnTypeError(
f"function `{func.__name__}` return type = `{type(component)}`",
"make sure that functions with @cell decorator return a Component",
)
if autoname and getattr(component, "_autoname", True):
component.name = name_component
# docstring = func.__doc__ if hasattr(func, "__doc__") else func.func.__doc__
# component.info.doc = docstring
component.info.module = func.__module__
component.info.function_name = func.__name__
component.info.info_version = INFO_VERSION
component.info.name_long = name_long
component.info.name = component.name
component.info.update(**info)
default = {
p.name: p.default
for p in sig.parameters.values()
if not callable(p.default)
}
full = default.copy()
full.update(**args_as_kwargs)
changed = args_as_kwargs.copy()
clean_dict(full)
clean_dict(default)
clean_dict(changed)
component.info.changed = changed
component.info.default = default
component.info.full = full
CACHE[name] = component
return component
return
def from_yaml(
yaml_str: Union[str, pathlib.Path, IO[Any]],
component_factory: ComponentFactoryDict = factory,
routing_strategy: Dict[str, Callable] = routing_strategy_factories,
cross_section_factory: Dict[str,
CrossSectionFactory] = cross_section_factory,
label_instance_function: Callable = add_instance_label,
**kwargs,
) -> Component:
"""Returns a Component defined in YAML file or string.
Args:
yaml: YAML IO describing Component file or string (with newlines)
(instances, placements, routes, ports, connections, names)
component_factory: dict of functions {factory_name: factory_function}
routing_strategy: for links
label_instance_function: to label each instance
kwargs: cache, prefix, autoname ... to pass to all factories
Returns:
Component
.. code::
valid properties:
name: Optional Component name
vars: Optional variables
info: Optional component info
description: just a demo
polarization: TE
...
instances:
name:
component: (ComponentFactory)
settings (Optional)
length: 10
...
placements:
x: Optional[float, str] str can be instanceName,portName
y: Optional[float, str]
rotation: Optional[float]
mirror: Optional[bool, float] float is x mirror axis
port: Optional[str] port anchor
connections (Optional): between instances
ports (Optional): ports to expose
routes (Optional): bundles of routes
routeName:
library: optical
links:
instance1,port1: instance2,port2
.. code::
vars:
lenght: 3
instances:
mmi_bot:
component: mmi1x2
settings:
width_mmi: 4.5
length_mmi: 10
mmi_top:
component: mmi1x2
settings:
width_mmi: 4.5
length_mmi: 5
placements:
mmi_top:
port: o1
x: 0
y: 0
mmi_bot:
port: o1
x: mmi_top,o2
y: mmi_top,o2
dx: 30
dy: -30
routes:
optical:
library: optical
links:
mmi_top,o3: mmi_bot,o1
"""
yaml_str = (io.StringIO(yaml_str) if isinstance(yaml_str, str)
and "\n" in yaml_str else yaml_str)
conf = OmegaConf.load(
yaml_str) # nicer loader than conf = yaml.safe_load(yaml_str)
for key in conf.keys():
assert key in valid_top_level_keys, f"{key} not in {list(valid_top_level_keys)}"
instances = {}
routes = {}
name = conf.get(
"name",
f"Unnamed_{hashlib.md5(json.dumps(OmegaConf.to_container(conf)).encode()).hexdigest()[:8]}",
)
if name in CACHE:
return CACHE[name]
else:
c = Component(name)
CACHE[name] = c
placements_conf = conf.get("placements")
routes_conf = conf.get("routes")
ports_conf = conf.get("ports")
connections_conf = conf.get("connections")
instances_dict = conf["instances"]
c.info = conf.get("info", omegaconf.DictConfig({}))
for instance_name in instances_dict:
instance_conf = instances_dict[instance_name]
component_type = instance_conf["component"]
assert (component_type in component_factory
), f"{component_type} not in {list(component_factory.keys())}"
settings = instance_conf.get("settings", {})
settings = OmegaConf.to_container(settings,
resolve=True) if settings else {}
settings.update(**kwargs)
if "cross_section" in settings:
name_or_dict = settings["cross_section"]
if isinstance(name_or_dict, str):
cross_section = cross_section_factory[name_or_dict]
elif isinstance(name_or_dict, dict):
name = name_or_dict.pop("function")
cross_section = functools.partial(cross_section_factory[name],
**name_or_dict)
else:
raise ValueError(
f"invalid type for cross_section={name_or_dict}")
settings["cross_section"] = cross_section
ci = component_factory[component_type](**settings)
ref = c << ci
instances[instance_name] = ref
placements_conf = dict() if placements_conf is None else placements_conf
connections_by_transformed_inst = transform_connections_dict(
connections_conf)
components_to_place = set(placements_conf.keys())
components_with_placement_conflicts = components_to_place.intersection(
connections_by_transformed_inst.keys())
for instance_name in components_with_placement_conflicts:
placement_settings = placements_conf[instance_name]
if "x" in placement_settings or "y" in placement_settings:
warnings.warn(
f"YAML defined: ({', '.join(components_with_placement_conflicts)}) "
+
"with both connection and placement. Please use one or the other.",
)
all_remaining_insts = list(
set(placements_conf.keys()).union(
set(connections_by_transformed_inst.keys())))
while all_remaining_insts:
place(
placements_conf=placements_conf,
connections_by_transformed_inst=connections_by_transformed_inst,
instances=instances,
encountered_insts=list(),
all_remaining_insts=all_remaining_insts,
)
for instance_name in instances_dict:
label_instance_function(component=c,
instance_name=instance_name,
reference=instances[instance_name])
if routes_conf:
for route_alias in routes_conf:
route_names = []
ports1 = []
ports2 = []
routes_dict = routes_conf[route_alias]
for key in routes_dict.keys():
if key not in valid_route_keys:
raise ValueError(
f"`{route_alias}` key=`{key}` not in {valid_route_keys}"
)
settings = routes_dict.pop("settings", {})
settings = (OmegaConf.to_container(settings, resolve=True)
if settings else {})
if "cross_section" in settings:
name_or_dict = settings["cross_section"]
if isinstance(name_or_dict, str):
cross_section = cross_section_factory[name_or_dict]
elif isinstance(name_or_dict, dict):
name = name_or_dict.pop("function")
cross_section = functools.partial(
cross_section_factory[name], **name_or_dict)
else:
raise ValueError(
f"invalid type for cross_section={name_or_dict}")
settings["cross_section"] = cross_section
routing_strategy_name = routes_dict.pop("routing_strategy",
"get_bundle")
if routing_strategy_name not in routing_strategy:
raise ValueError(
f"function `{routing_strategy_name}` not in routing_strategy {list(routing_strategy.keys())}"
)
if "links" not in routes_dict:
raise ValueError(
f"You need to define links for the `{route_alias}` route")
links_dict = routes_dict["links"]
for port_src_string, port_dst_string in links_dict.items():
if ":" in port_src_string:
src, src0, src1 = [
s.strip() for s in port_src_string.split(":")
]
dst, dst0, dst1 = [
s.strip() for s in port_dst_string.split(":")
]
instance_src_name, port_src_name = [
s.strip() for s in src.split(",")
]
instance_dst_name, port_dst_name = [
s.strip() for s in dst.split(",")
]
src0 = int(src0)
src1 = int(src1)
dst0 = int(dst0)
dst1 = int(dst1)
if src1 > src0:
ports1names = [
f"{port_src_name}{i}"
for i in range(src0, src1 + 1, 1)
]
else:
ports1names = [
f"{port_src_name}{i}"
for i in range(src0, src1 - 1, -1)
]
if dst1 > dst0:
ports2names = [
f"{port_dst_name}{i}"
for i in range(dst0, dst1 + 1, 1)
]
else:
ports2names = [
f"{port_dst_name}{i}"
for i in range(dst0, dst1 - 1, -1)
]
assert len(ports1names) == len(ports2names)
route_names += [
f"{instance_src_name},{i}:{instance_dst_name},{j}"
for i, j in zip(ports1names, ports2names)
]
instance_src = instances[instance_src_name]
instance_dst = instances[instance_dst_name]
for port_src_name in ports1names:
assert port_src_name in instance_src.ports, (
f"{port_src_name} not in {list(instance_src.ports.keys())}"
f"for {instance_src_name} ")
ports1.append(instance_src.ports[port_src_name])
for port_dst_name in ports2names:
assert port_dst_name in instance_dst.ports, (
f"{port_dst_name} not in {list(instance_dst.ports.keys())}"
f"for {instance_dst_name}")
ports2.append(instance_dst.ports[port_dst_name])
# print(ports1)
# print(ports2)
# print(route_names)
else:
instance_src_name, port_src_name = port_src_string.split(
",")
instance_dst_name, port_dst_name = port_dst_string.split(
",")
instance_src_name = instance_src_name.strip()
instance_dst_name = instance_dst_name.strip()
port_src_name = port_src_name.strip()
port_dst_name = port_dst_name.strip()
assert (
instance_src_name in instances
), f"{instance_src_name} not in {list(instances.keys())}"
assert (
instance_dst_name in instances
), f"{instance_dst_name} not in {list(instances.keys())}"
instance_src = instances[instance_src_name]
instance_dst = instances[instance_dst_name]
assert port_src_name in instance_src.ports, (
f"{port_src_name} not in {list(instance_src.ports.keys())} for"
f" {instance_src_name} ")
assert port_dst_name in instance_dst.ports, (
f"{port_dst_name} not in {list(instance_dst.ports.keys())} for"
f" {instance_dst_name}")
ports1.append(instance_src.ports[port_src_name])
ports2.append(instance_dst.ports[port_dst_name])
route_name = f"{port_src_string}:{port_dst_string}"
route_names.append(route_name)
routing_function = routing_strategy[routing_strategy_name]
route_or_route_list = routing_function(
ports1=ports1,
ports2=ports2,
**settings,
)
# FIXME, be more consistent
if isinstance(route_or_route_list, list):
for route_name, route_dict in zip(route_names,
route_or_route_list):
c.add(route_dict.references)
routes[route_name] = route_dict.length
elif isinstance(route_or_route_list, Route):
c.add(route_or_route_list.references)
routes[route_name] = route_or_route_list.length
else:
raise ValueError(
f"{route_or_route_list} needs to be a Route or a list")
if ports_conf:
assert hasattr(ports_conf, "items"), f"{ports_conf} needs to be a dict"
for port_name, instance_comma_port in ports_conf.items():
instance_name, instance_port_name = instance_comma_port.split(",")
instance_name = instance_name.strip()
instance_port_name = instance_port_name.strip()
assert (instance_name in instances
), f"{instance_name} not in {list(instances.keys())}"
instance = instances[instance_name]
assert instance_port_name in instance.ports, (
f"{instance_port_name} not in {list(instance.ports.keys())} for"
f" {instance_name} ")
c.add_port(port_name, port=instance.ports[instance_port_name])
c.routes = routes
c.instances = instances
return c
def config(project_config, template_config):
return omegaconf.DictConfig({"project": project_config, "template": template_config})
def get_netlist(
component: Component,
full_settings: bool = False,
layer_label: Tuple[int, int] = LAYER.LABEL_INSTANCE,
tolerance: int = 1,
) -> omegaconf.DictConfig:
"""From a component returns instances, connections and placements dict. It
assumes that ports with same width, x, y are connected.
Args:
component: to Extract netlist
full_settings: True returns all settings, false only the ones that have changed
layer_label: label to read instanceNames from (if any)
tolerance: tolerance in nm to consider two ports the same
Returns:
connections: Dict of Instance1Name,portName: Instace2Name,portName
instances: Dict of instances and settings
placements: Dict of instances and placements (x, y, rotation)
port: Dict portName: CompoentName,port
name: name of component
"""
placements = {}
instances = {}
connections = {}
top_ports = {}
for reference in component.references:
c = reference.parent
origin = reference.origin
x = float(snap_to_grid(origin[0]))
y = float(snap_to_grid(origin[1]))
reference_name = get_instance_name(component,
reference,
layer_label=layer_label)
settings = (c.info.get("full", {}) if full_settings else c.info.get(
"changed", {}))
instances[reference_name] = dict(
component=getattr(c.info, "function_name", c.name),
settings=settings,
)
placements[reference_name] = dict(
x=x,
y=y,
rotation=int(reference.rotation),
mirror=reference.x_reflection,
)
# store where ports are located
name2port = {}
# Initialize a dict of port locations to Instance1Name,PortNames
port_locations = {}
# TOP level ports
ports = component.get_ports(depth=0)
top_ports_list = set()
for port in ports:
src = port.name
name2port[src] = port
top_ports_list.add(src)
# lower level ports
for reference in component.references:
for port in reference.ports.values():
reference_name = get_instance_name(component,
reference,
layer_label=layer_label)
src = f"{reference_name},{port.name}"
name2port[src] = port
# build connectivity port_locations = Dict[Tuple(x,y,width), set of portNames]
for name, port in name2port.items():
xyw = tuple(
round(1000 * v)
for v in snap_to_grid((port.x, port.y, port.width), nm=tolerance))
if xyw not in port_locations:
port_locations[xyw] = set()
port_locations[xyw].add(name)
for xyw, names_set in port_locations.items():
if len(names_set) > 2:
x, y, w = (v / 1000 for v in xyw)
raise ValueError(
f"more than 2 connections at {x, y} {list(names_set)}, width = {w} "
)
if len(names_set) == 2:
names_list = list(names_set)
src = names_list[0]
dst = names_list[1]
if src in top_ports_list:
top_ports[src] = dst
elif dst in top_ports_list:
top_ports[dst] = src
else:
src_dest = sorted([src, dst])
connections[src_dest[0]] = src_dest[1]
connections_sorted = {
k: connections[k]
for k in sorted(list(connections.keys()))
}
placements_sorted = {
k: placements[k]
for k in sorted(list(placements.keys()))
}
instances_sorted = {
k: instances[k]
for k in sorted(list(instances.keys()))
}
return omegaconf.DictConfig(
dict(
connections=connections_sorted,
instances=instances_sorted,
placements=placements_sorted,
ports=top_ports,
name=component.name,
))