def _make_config(self, data=None, key=None, val=None, obj=None):
config = obj or wandb_internal_pb2.ConfigRecord()
if data:
for k, v in six.iteritems(data):
update = config.update.add()
update.key = k
update.value_json = json_dumps_safer(json_friendly(v)[0])
if key:
update = config.update.add()
if isinstance(key, tuple):
update.nested_key = key
else:
update.key = key
update.value_json = json_dumps_safer(json_friendly(val)[0])
return config
def _write(self, commit=False):
mutation = gql('''
mutation UpsertBucket( $id: String, $summaryMetrics: JSONString) {
upsertBucket(input: { id: $id, summaryMetrics: $summaryMetrics}) {
bucket { id }
}
}
''')
if commit:
if self._h5:
self._h5.close()
self._h5 = None
res = self._client.execute(mutation,
variable_values={
'id':
self._run.storage_id,
'summaryMetrics':
util.json_dumps_safer(
self._json_dict)
})
assert res['upsertBucket']['bucket']['id']
entity, project, run = self._run.path
if os.path.exists(self._h5_path) and os.path.getmtime(
self._h5_path) >= self._started:
upload_h5(self._h5_path, run, entity=entity, project=project)
else:
return False
def _make_config(self, config_dict, obj=None):
config = obj or wandb_internal_pb2.ConfigRecord()
for k, v in six.iteritems(config_dict):
update = config.update.add()
update.key = k
update.value_json = json_dumps_safer(json_friendly(v)[0])
return config
def _make_stats(self, stats_dict):
stats = wandb_internal_pb2.StatsRecord()
stats.stats_type = wandb_internal_pb2.StatsRecord.StatsType.SYSTEM
stats.timestamp.GetCurrentTime()
for k, v in six.iteritems(stats_dict):
item = stats.item.add()
item.key = k
item.value_json = json_dumps_safer(json_friendly(v)[0])
return stats
def _make_stats(self, stats_dict: dict) -> pb.StatsRecord:
stats = pb.StatsRecord()
stats.stats_type = pb.StatsRecord.StatsType.SYSTEM
stats.timestamp.GetCurrentTime()
for k, v in six.iteritems(stats_dict):
item = stats.item.add()
item.key = k
item.value_json = json_dumps_safer(json_friendly(v)[0]) # type: ignore
return stats
def write(self):
self.lock.acquire()
try:
self.data["heartbeatAt"] = datetime.utcnow().isoformat()
with open(self.fname, 'w') as f:
s = util.json_dumps_safer(self.data, indent=4)
f.write(s)
f.write('\n')
finally:
self.lock.release()
def _write(self, commit=False):
# TODO: we just ignore commit to ensure backward capability
with open(self._fname, 'w') as f:
s = util.json_dumps_safer(self._json_dict, indent=4)
f.write(s)
f.write('\n')
f.flush()
os.fsync(f.fileno())
if self._h5:
self._h5.close()
self._h5 = None
def _write(self, commit=False):
# TODO: we just ignore commit to ensure backward capability
with open(self._fname, 'w') as f:
f.write(util.json_dumps_safer(self._json_dict))
f.write('\n')
f.flush()
os.fsync(f.fileno())
if self._h5:
self._h5.close()
self._h5 = None
if wandb.run and wandb.run._jupyter_agent:
wandb.run._jupyter_agent.start()
def _make_config(
self,
data = None,
key = None,
val = None,
obj = None,
):
config = obj or pb.ConfigRecord()
if data:
for k, v in six.iteritems(data):
update = config.update.add()
update.key = k
update.value_json = json_dumps_safer(json_friendly(v)[0]) # type: ignore
if key:
update = config.update.add()
if isinstance(key, tuple):
for k in key:
update.nested_key.append(k)
else:
update.key = key
update.value_json = json_dumps_safer(json_friendly(val)[0]) # type: ignore
return config
def _make_config(
self,
data: dict = None,
key: Union[Tuple[str, ...], str] = None,
val: Any = None,
obj: pb.ConfigRecord = None,
) -> pb.ConfigRecord:
config = obj or pb.ConfigRecord()
if data:
for k, v in six.iteritems(data):
update = config.update.add()
update.key = k
update.value_json = json_dumps_safer(json_friendly(v)[0])
if key:
update = config.update.add()
if isinstance(key, tuple):
for k in key:
update.nested_key.append(k)
else:
update.key = key
update.value_json = json_dumps_safer(json_friendly(val)[0])
return config
def track(self, event, properties, timestamp=None, _wandb=False):
if not isinstance(properties, collections.Mapping):
raise wandb.Error('event.track expects dict-like object')
self.lock.acquire()
try:
row = {}
row[event] = properties
self.flatten(row)
if _wandb:
row["_wandb"] = _wandb
row["_timestamp"] = int(timestamp or time.time())
row['_runtime'] = int(time.time() - self._start_time)
self._file.write(util.json_dumps_safer(row))
self._file.write('\n')
self._file.flush()
finally:
self.lock.release()
def _write(self):
if self.row:
self._lock.acquire()
try:
self.row['_runtime'] = time.time() - self._start_time
self.row['_timestamp'] = time.time()
self.row['_step'] = self._steps
if self.stream_name != "default":
self.row["_stream"] = self.stream_name
self._transform()
self._file.write(util.json_dumps_safer(self.row))
self._file.write('\n')
self._file.flush()
self._index(self.row)
if self._add_callback:
self._add_callback(self.row)
self.row = {}
finally:
self._lock.release()
return True
else:
return False
def from_directory(cls,
directory,
project=None,
entity=None,
run_id=None,
api=None,
ignore_globs=None):
api = api or InternalApi()
run_id = run_id or util.generate_id()
run = Run(run_id=run_id, dir=directory)
run_name = None
project_from_meta = None
snap = DirectorySnapshot(directory)
meta = next((p for p in snap.paths if METADATA_FNAME in p), None)
if meta:
meta = json.load(open(meta))
run_name = meta.get("name")
project_from_meta = meta.get("project")
project = project or project_from_meta or api.settings(
"project") or run.auto_project_name(api=api)
if project is None:
raise ValueError("You must specify project")
api.set_current_run_id(run_id)
api.set_setting("project", project)
if entity:
api.set_setting("entity", entity)
res = api.upsert_run(name=run_id,
project=project,
entity=entity,
display_name=run_name)
entity = res["project"]["entity"]["name"]
wandb.termlog("Syncing {} to:".format(directory))
try:
wandb.termlog(res["displayName"] + " " + run.get_url(api))
except CommError as e:
wandb.termwarn(e.message)
file_api = api.get_file_stream_api()
file_api.start()
paths = [
os.path.relpath(abs_path, directory) for abs_path in snap.paths
if os.path.isfile(abs_path)
]
if ignore_globs:
paths = set(paths)
for g in ignore_globs:
paths = paths - set(fnmatch.filter(paths, g))
paths = list(paths)
run_update = {"id": res["id"]}
tfevents = sorted([p for p in snap.paths if ".tfevents." in p])
history = next((p for p in snap.paths if HISTORY_FNAME in p), None)
event = next((p for p in snap.paths if EVENTS_FNAME in p), None)
config = next((p for p in snap.paths if CONFIG_FNAME in p), None)
user_config = next((p for p in snap.paths if USER_CONFIG_FNAME in p),
None)
summary = next((p for p in snap.paths if SUMMARY_FNAME in p), None)
if history:
wandb.termlog("Uploading history metrics")
file_api.stream_file(history)
snap.paths.remove(history)
elif len(tfevents) > 0:
from wandb import tensorflow as wbtf
wandb.termlog("Found tfevents file, converting...")
summary = {}
for path in tfevents:
filename = os.path.basename(path)
namespace = path.replace(filename,
"").replace(directory,
"").strip(os.sep)
summary.update(
wbtf.stream_tfevents(path,
file_api,
run,
namespace=namespace))
for path in glob.glob(os.path.join(directory, "media/**/*"),
recursive=True):
if os.path.isfile(path):
paths.append(path)
else:
wandb.termerror(
"No history or tfevents files found, only syncing files")
if event:
file_api.stream_file(event)
snap.paths.remove(event)
if config:
run_update["config"] = util.load_yaml(open(config))
elif user_config:
# TODO: half backed support for config.json
run_update["config"] = {
k: {
"value": v
}
for k, v in six.iteritems(user_config)
}
if isinstance(summary, dict):
#TODO: summary should already have data_types converted here...
run_update["summary_metrics"] = util.json_dumps_safer(summary)
elif summary:
run_update["summary_metrics"] = open(summary).read()
if meta:
if meta.get("git"):
run_update["commit"] = meta["git"].get("commit")
run_update["repo"] = meta["git"].get("remote")
if meta.get("host"):
run_update["host"] = meta["host"]
run_update["program_path"] = meta["program"]
run_update["job_type"] = meta.get("jobType")
run_update["notes"] = meta.get("notes")
else:
run_update["host"] = run.host
wandb.termlog("Updating run and uploading files")
api.upsert_run(**run_update)
pusher = FilePusher(api)
for k in paths:
path = os.path.abspath(os.path.join(directory, k))
pusher.update_file(k, path)
pusher.file_changed(k, path)
pusher.finish()
pusher.print_status()
file_api.finish(0)
# Remove temporary media images generated from tfevents
if history is None and os.path.exists(os.path.join(directory,
"media")):
shutil.rmtree(os.path.join(directory, "media"))
wandb.termlog("Finished!")
return run
def _write(self):
with open(self._fname, 'w') as f:
s = util.json_dumps_safer(self._summary, indent=4)
f.write(s)
f.write('\n')
