def _make_config(self, data=None, key=None, val=None, obj=None): config = obj or wandb_internal_pb2.ConfigRecord() if data: for k, v in six.iteritems(data): update = config.update.add() update.key = k update.value_json = json_dumps_safer(json_friendly(v)[0]) if key: update = config.update.add() if isinstance(key, tuple): update.nested_key = key else: update.key = key update.value_json = json_dumps_safer(json_friendly(val)[0]) return config
def _write(self, commit=False): mutation = gql(''' mutation UpsertBucket( $id: String, $summaryMetrics: JSONString) { upsertBucket(input: { id: $id, summaryMetrics: $summaryMetrics}) { bucket { id } } } ''') if commit: if self._h5: self._h5.close() self._h5 = None res = self._client.execute(mutation, variable_values={ 'id': self._run.storage_id, 'summaryMetrics': util.json_dumps_safer( self._json_dict) }) assert res['upsertBucket']['bucket']['id'] entity, project, run = self._run.path if os.path.exists(self._h5_path) and os.path.getmtime( self._h5_path) >= self._started: upload_h5(self._h5_path, run, entity=entity, project=project) else: return False
def _make_config(self, config_dict, obj=None): config = obj or wandb_internal_pb2.ConfigRecord() for k, v in six.iteritems(config_dict): update = config.update.add() update.key = k update.value_json = json_dumps_safer(json_friendly(v)[0]) return config
def _make_stats(self, stats_dict): stats = wandb_internal_pb2.StatsRecord() stats.stats_type = wandb_internal_pb2.StatsRecord.StatsType.SYSTEM stats.timestamp.GetCurrentTime() for k, v in six.iteritems(stats_dict): item = stats.item.add() item.key = k item.value_json = json_dumps_safer(json_friendly(v)[0]) return stats
def _make_stats(self, stats_dict: dict) -> pb.StatsRecord: stats = pb.StatsRecord() stats.stats_type = pb.StatsRecord.StatsType.SYSTEM stats.timestamp.GetCurrentTime() for k, v in six.iteritems(stats_dict): item = stats.item.add() item.key = k item.value_json = json_dumps_safer(json_friendly(v)[0]) # type: ignore return stats
def write(self): self.lock.acquire() try: self.data["heartbeatAt"] = datetime.utcnow().isoformat() with open(self.fname, 'w') as f: s = util.json_dumps_safer(self.data, indent=4) f.write(s) f.write('\n') finally: self.lock.release()
def _write(self, commit=False): # TODO: we just ignore commit to ensure backward capability with open(self._fname, 'w') as f: s = util.json_dumps_safer(self._json_dict, indent=4) f.write(s) f.write('\n') f.flush() os.fsync(f.fileno()) if self._h5: self._h5.close() self._h5 = None
def _write(self, commit=False): # TODO: we just ignore commit to ensure backward capability with open(self._fname, 'w') as f: f.write(util.json_dumps_safer(self._json_dict)) f.write('\n') f.flush() os.fsync(f.fileno()) if self._h5: self._h5.close() self._h5 = None if wandb.run and wandb.run._jupyter_agent: wandb.run._jupyter_agent.start()
def _make_config( self, data = None, key = None, val = None, obj = None, ): config = obj or pb.ConfigRecord() if data: for k, v in six.iteritems(data): update = config.update.add() update.key = k update.value_json = json_dumps_safer(json_friendly(v)[0]) # type: ignore if key: update = config.update.add() if isinstance(key, tuple): for k in key: update.nested_key.append(k) else: update.key = key update.value_json = json_dumps_safer(json_friendly(val)[0]) # type: ignore return config
def _make_config( self, data: dict = None, key: Union[Tuple[str, ...], str] = None, val: Any = None, obj: pb.ConfigRecord = None, ) -> pb.ConfigRecord: config = obj or pb.ConfigRecord() if data: for k, v in six.iteritems(data): update = config.update.add() update.key = k update.value_json = json_dumps_safer(json_friendly(v)[0]) if key: update = config.update.add() if isinstance(key, tuple): for k in key: update.nested_key.append(k) else: update.key = key update.value_json = json_dumps_safer(json_friendly(val)[0]) return config
def track(self, event, properties, timestamp=None, _wandb=False): if not isinstance(properties, collections.Mapping): raise wandb.Error('event.track expects dict-like object') self.lock.acquire() try: row = {} row[event] = properties self.flatten(row) if _wandb: row["_wandb"] = _wandb row["_timestamp"] = int(timestamp or time.time()) row['_runtime'] = int(time.time() - self._start_time) self._file.write(util.json_dumps_safer(row)) self._file.write('\n') self._file.flush() finally: self.lock.release()
def _write(self): if self.row: self._lock.acquire() try: self.row['_runtime'] = time.time() - self._start_time self.row['_timestamp'] = time.time() self.row['_step'] = self._steps if self.stream_name != "default": self.row["_stream"] = self.stream_name self._transform() self._file.write(util.json_dumps_safer(self.row)) self._file.write('\n') self._file.flush() self._index(self.row) if self._add_callback: self._add_callback(self.row) self.row = {} finally: self._lock.release() return True else: return False
def from_directory(cls, directory, project=None, entity=None, run_id=None, api=None, ignore_globs=None): api = api or InternalApi() run_id = run_id or util.generate_id() run = Run(run_id=run_id, dir=directory) run_name = None project_from_meta = None snap = DirectorySnapshot(directory) meta = next((p for p in snap.paths if METADATA_FNAME in p), None) if meta: meta = json.load(open(meta)) run_name = meta.get("name") project_from_meta = meta.get("project") project = project or project_from_meta or api.settings( "project") or run.auto_project_name(api=api) if project is None: raise ValueError("You must specify project") api.set_current_run_id(run_id) api.set_setting("project", project) if entity: api.set_setting("entity", entity) res = api.upsert_run(name=run_id, project=project, entity=entity, display_name=run_name) entity = res["project"]["entity"]["name"] wandb.termlog("Syncing {} to:".format(directory)) try: wandb.termlog(res["displayName"] + " " + run.get_url(api)) except CommError as e: wandb.termwarn(e.message) file_api = api.get_file_stream_api() file_api.start() paths = [ os.path.relpath(abs_path, directory) for abs_path in snap.paths if os.path.isfile(abs_path) ] if ignore_globs: paths = set(paths) for g in ignore_globs: paths = paths - set(fnmatch.filter(paths, g)) paths = list(paths) run_update = {"id": res["id"]} tfevents = sorted([p for p in snap.paths if ".tfevents." in p]) history = next((p for p in snap.paths if HISTORY_FNAME in p), None) event = next((p for p in snap.paths if EVENTS_FNAME in p), None) config = next((p for p in snap.paths if CONFIG_FNAME in p), None) user_config = next((p for p in snap.paths if USER_CONFIG_FNAME in p), None) summary = next((p for p in snap.paths if SUMMARY_FNAME in p), None) if history: wandb.termlog("Uploading history metrics") file_api.stream_file(history) snap.paths.remove(history) elif len(tfevents) > 0: from wandb import tensorflow as wbtf wandb.termlog("Found tfevents file, converting...") summary = {} for path in tfevents: filename = os.path.basename(path) namespace = path.replace(filename, "").replace(directory, "").strip(os.sep) summary.update( wbtf.stream_tfevents(path, file_api, run, namespace=namespace)) for path in glob.glob(os.path.join(directory, "media/**/*"), recursive=True): if os.path.isfile(path): paths.append(path) else: wandb.termerror( "No history or tfevents files found, only syncing files") if event: file_api.stream_file(event) snap.paths.remove(event) if config: run_update["config"] = util.load_yaml(open(config)) elif user_config: # TODO: half backed support for config.json run_update["config"] = { k: { "value": v } for k, v in six.iteritems(user_config) } if isinstance(summary, dict): #TODO: summary should already have data_types converted here... run_update["summary_metrics"] = util.json_dumps_safer(summary) elif summary: run_update["summary_metrics"] = open(summary).read() if meta: if meta.get("git"): run_update["commit"] = meta["git"].get("commit") run_update["repo"] = meta["git"].get("remote") if meta.get("host"): run_update["host"] = meta["host"] run_update["program_path"] = meta["program"] run_update["job_type"] = meta.get("jobType") run_update["notes"] = meta.get("notes") else: run_update["host"] = run.host wandb.termlog("Updating run and uploading files") api.upsert_run(**run_update) pusher = FilePusher(api) for k in paths: path = os.path.abspath(os.path.join(directory, k)) pusher.update_file(k, path) pusher.file_changed(k, path) pusher.finish() pusher.print_status() file_api.finish(0) # Remove temporary media images generated from tfevents if history is None and os.path.exists(os.path.join(directory, "media")): shutil.rmtree(os.path.join(directory, "media")) wandb.termlog("Finished!") return run
def _write(self): with open(self._fname, 'w') as f: s = util.json_dumps_safer(self._summary, indent=4) f.write(s) f.write('\n')