def main():
# Use concurrency experiment
wandb.require(experiment="service")
print("PIDPID", os.getpid())
# Set up data
num_samples = 100000
train = RandomDataset(32, num_samples)
train = DataLoader(train, batch_size=32)
val = RandomDataset(32, num_samples)
val = DataLoader(val, batch_size=32)
test = RandomDataset(32, num_samples)
test = DataLoader(test, batch_size=32)
# init model
model = BoringModel()
# set up wandb
config = dict(some_hparam="Logged Before Trainer starts DDP")
wandb_logger = WandbLogger(log_model=True, config=config, save_code=True)
# Initialize a trainer
trainer = pl.Trainer(
max_epochs=1,
progress_bar_refresh_rate=20,
num_processes=2,
accelerator="ddp_cpu",
logger=wandb_logger,
)
# Train the model
trainer.fit(model, train, val)
trainer.test(dataloaders=test)
def main():
# Use concurrency experiment
wandb.require(experiment="service")
print("PIDPID", os.getpid())
# Set up data
num_samples = 100000
train = DataLoader(RandomDataset(32, num_samples), batch_size=32)
val = DataLoader(RandomDataset(32, num_samples), batch_size=32)
test = DataLoader(RandomDataset(32, num_samples), batch_size=32)
# init model
model = BoringModel()
# set up wandb
config = dict(some_hparam="Logged Before Trainer starts DDP")
wandb_logger = WandbLogger(log_model=True, config=config, save_code=True)
# Initialize a trainer
trainer = Trainer(
max_epochs=1,
gpus=2,
strategy="ddp",
logger=wandb_logger,
)
# Train the model
trainer.fit(model, train, val)
trainer.test(test_dataloaders=test)
def test_require_good(user_test, require_mock):
def mock_require_test(self):
wandb.require_test = lambda x: x + 2
require_mock("test", mock_require_test)
wandb.require("test")
assert wandb.require_test(2) == 4
def __init__(
self,
name: Optional[str] = None,
save_dir: Optional[str] = None,
offline: Optional[bool] = False,
id: Optional[str] = None,
anonymous: Optional[bool] = None,
version: Optional[str] = None,
project: Optional[str] = None,
log_model: Union[str, bool] = False,
experiment=None,
prefix: Optional[str] = "",
**kwargs,
):
if wandb is None:
raise ModuleNotFoundError(
"You want to use `wandb` logger which is not installed yet,"
" install it with `pip install wandb`." # pragma: no-cover
)
if offline and log_model:
raise MisconfigurationException(
f"Providing log_model={log_model} and offline={offline} is an invalid configuration"
" since model checkpoints cannot be uploaded in offline mode.\n"
"Hint: Set `offline=False` to log your model.")
if log_model and not _WANDB_GREATER_EQUAL_0_10_22:
rank_zero_warn(
f"Providing log_model={log_model} requires wandb version >= 0.10.22"
" for logging associated model metadata.\n"
"Hint: Upgrade with `pip install --upgrade wandb`.")
super().__init__()
self._offline = offline
self._log_model = log_model
self._prefix = prefix
self._experiment = experiment
self._logged_model_time = {}
self._checkpoint_callback = None
# set wandb init arguments
anonymous_lut = {True: "allow", False: None}
self._wandb_init = dict(
name=name,
project=project,
id=version or id,
dir=save_dir,
resume="allow",
anonymous=anonymous_lut.get(anonymous, anonymous),
)
self._wandb_init.update(**kwargs)
# extract parameters
self._save_dir = self._wandb_init.get("dir")
self._name = self._wandb_init.get("name")
self._id = self._wandb_init.get("id")
# start wandb run (to create an attach_id for distributed modes)
if _WANDB_GREATER_EQUAL_0_12_10:
wandb.require("service")
_ = self.experiment
def main():
wandb.require("service")
wandb.setup()
num_proc = 4
pool = mp.Pool(processes=num_proc)
result = pool.map_async(do_run, range(num_proc))
data = result.get(60)
print(f"DEBUG: {data}")
assert len(data) == 4
def test_require_require(user_test, require_mock):
def mock_require_test(self):
wandb.require_test = lambda x: x + 2
require_mock("test", mock_require_test)
# remove autofixture
wandb.__dict__.pop("require", None)
wandb._require("require")
wandb.require("test")
assert wandb.require_test(2) == 4
def main():
wandb.require("service")
run = wandb.init()
run.config.c1 = 11
run.log({"s1": 11})
run.log({"s2": 12})
run.log({"s2": 120})
# Start a new run in parallel in a child process
p = mp.Process(target=process_child, kwargs=dict(run=run))
p.start()
p.join()
run.log({"s3": 13})
def main():
wandb.require("service")
run = wandb.init()
print("parent output")
run.config.c1 = 11
run.log(dict(s2=12, s4=14))
# Start a new run in parallel in a child process
attach_id = run.id
p = mp.Process(target=process_child, kwargs=dict(attach_id=attach_id))
p.start()
p.join()
# run can still be logged to after join (and eventually anytime?)
run.log(dict(s3=13))
print("more output from parent")
def main():
wandb.require("service")
run = wandb.init()
run.log(dict(m1=1))
run.log(dict(m2=2))
with open("my-dataset.txt", "w") as fp:
fp.write("this-is-data")
artifact = wandb.Artifact('my-dataset', type='dataset')
table = wandb.Table(columns=["a", "b", "c"], data=[[1, 2, 3]])
artifact.add(table, "my_table")
artifact.add_file('my-dataset.txt')
art = run.log_artifact(artifact)
art.wait()
get_table = art.get("my_table")
print("TABLE", get_table)
run.finish()
def run(self):
wandb.require("service")
wandb.init(*self.args, **self.kwargs)
wandb.setup()
while True:
item_type, item_content = self.queue.get()
if item_type == _QueueItem.END:
break
if item_type == _QueueItem.CHECKPOINT:
self._handle_checkpoint(item_content)
continue
assert item_type == _QueueItem.RESULT
log, config_update = self._handle_result(item_content)
try:
wandb.config.update(config_update, allow_val_change=True)
wandb.log(log)
except urllib.error.HTTPError as e:
# Ignore HTTPError. Missing a few data points is not a
# big issue, as long as things eventually recover.
logger.warn("Failed to log result to w&b: {}".format(str(e)))
wandb.finish()
def main():
wandb.require("service")
try:
os.mkdir("tmp")
except FileExistsError:
pass
run_parent = wandb.init()
run_parent.config.id = "parent"
run_parent.log({"p1": 11})
run_parent.name = "parent-name"
fname1 = os.path.join("tmp", "03-parent-1.txt")
with open(fname1, "w") as fp:
fp.write("parent-1-data")
run_parent.save(fname1)
# Start a new run in parallel in a child process
p = mp.Process(target=process_child)
p.start()
fname2 = os.path.join("tmp", "03-parent-2.txt")
with open(fname2, "w") as fp:
fp.write("parent-2-data")
run_parent.save(fname2)
p.join()
fname3 = os.path.join("tmp", "03-parent-3.txt")
with open(fname3, "w") as fp:
fp.write("parent-3-data")
run_parent.save(fname3)
run_parent.log({"p1": 12})
run_parent.finish()
def test_require_require(user_test, require_mock):
# This is a noop now that it is "released"
wandb.require("require")
#!/usr/bin/env python
import wandb
wandb.require("service")
wandb.setup()
run = wandb.init()
run.log(dict(m1=1))
run.log(dict(m2=2))
wandb.teardown()
def __init__(
self,
name: Optional[str] = None,
save_dir: Optional[str] = None,
offline: bool = False,
id: Optional[str] = None,
anonymous: Optional[bool] = None,
version: Optional[str] = None,
project: Optional[str] = None,
log_model: Union[str, bool] = False,
experiment: Union[Run, RunDisabled, None] = None,
prefix: str = "",
agg_key_funcs: Optional[Mapping[str, Callable[[Sequence[float]],
float]]] = None,
agg_default_func: Optional[Callable[[Sequence[float]], float]] = None,
**kwargs: Any,
) -> None:
if wandb is None:
raise ModuleNotFoundError(
"You want to use `wandb` logger which is not installed yet,"
" install it with `pip install wandb`." # pragma: no-cover
)
if offline and log_model:
raise MisconfigurationException(
f"Providing log_model={log_model} and offline={offline} is an invalid configuration"
" since model checkpoints cannot be uploaded in offline mode.\n"
"Hint: Set `offline=False` to log your model.")
if log_model and not _WANDB_GREATER_EQUAL_0_10_22:
rank_zero_warn(
f"Providing log_model={log_model} requires wandb version >= 0.10.22"
" for logging associated model metadata.\n"
"Hint: Upgrade with `pip install --upgrade wandb`.")
super().__init__(agg_key_funcs=agg_key_funcs,
agg_default_func=agg_default_func)
self._offline = offline
self._log_model = log_model
self._prefix = prefix
self._experiment = experiment
self._logged_model_time: Dict[str, float] = {}
self._checkpoint_callback: Optional["ReferenceType[Checkpoint]"] = None
# set wandb init arguments
self._wandb_init: Dict[str, Any] = dict(
name=name or project,
project=project,
id=version or id,
dir=save_dir,
resume="allow",
anonymous=("allow" if anonymous else None),
)
self._wandb_init.update(**kwargs)
# extract parameters
self._save_dir = self._wandb_init.get("dir")
self._name = self._wandb_init.get("name")
self._id = self._wandb_init.get("id")
# start wandb run (to create an attach_id for distributed modes)
if _WANDB_GREATER_EQUAL_0_12_10:
wandb.require("service")
_ = self.experiment
def test_require_single(require_feature, user_test, capsys):
with pytest.raises(wandb.errors.RequireError):
wandb.require("something")
captured = capsys.readouterr()
assert "unsupported requirement: something" in captured.err
def test_require_single(user_test, capsys):
with pytest.raises(wandb.errors.RequireError):
wandb.require("something")
captured = capsys.readouterr()
assert "unsupported requirement: something" in captured.err
assert "http://wandb.me/library-require" in captured.err
def test_require_list(user_test, capsys):
with pytest.raises(wandb.errors.RequireError):
wandb.require(["something", "another"])
captured = capsys.readouterr()
assert "unsupported requirement: something" in captured.err
assert "unsupported requirement: another" in captured.err
def test_require_param(user_test, capsys):
with pytest.raises(wandb.errors.RequireError):
wandb.require("something:param@beta")
captured = capsys.readouterr()
assert "unsupported requirement: something" in captured.err
