def run(self):
for sync_item in self._sync_list:
dirname = os.path.dirname(sync_item)
files_dir = os.path.join(dirname, "files")
sd = dict(files_dir=files_dir,
_start_time=0,
)
settings = settings_static.SettingsStatic(sd)
resp_queue = queue.Queue()
sm = sender.SendManager(settings=settings, resp_q=resp_queue)
ds = datastore.DataStore()
ds.open_for_scan(sync_item)
while True:
data = ds.scan_data()
if data is None:
break
pb = wandb_internal_pb2.Record()
pb.ParseFromString(data)
sm.send(pb)
if pb.control.req_resp:
try:
_ = resp_queue.get(timeout=20)
except queue.Empty:
raise Exception("timeout?")
sm.finish()
def _setup(self):
self._sm = sender.SendManager(
settings=self._settings,
record_q=self._record_q,
result_q=self._result_q,
interface=self._interface,
)
def wandb_send(settings, q, resp_q, read_q, data_q, stopped):
sh = sender.SendManager(settings, resp_q)
while not stopped.isSet():
try:
i = q.get(timeout=1)
except queue.Empty:
continue
sh.send(i)
sh.finish()
def run(self):
for sync_item in self._sync_list:
if os.path.isdir(sync_item):
files = os.listdir(sync_item)
filtered_files = list(
filter(lambda f: f.endswith(WANDB_SUFFIX), files))
if check_and_warn_old(files) or len(filtered_files) != 1:
print("Skipping directory: {}".format(sync_item))
continue
sync_item = os.path.join(sync_item, filtered_files[0])
dirname = os.path.dirname(sync_item)
files_dir = os.path.join(dirname, "files")
sd = dict(
files_dir=files_dir,
_start_time=0,
git_remote=None,
resume=None,
program=None,
ignore_globs=(),
run_id=None,
entity=None,
project=None,
run_group=None,
job_type=None,
run_tags=None,
run_name=None,
run_notes=None,
save_code=None,
)
settings = settings_static.SettingsStatic(sd)
record_q = queue.Queue()
result_q = queue.Queue()
publish_interface = interface.BackendSender(record_q=record_q)
sm = sender.SendManager(
settings=settings,
record_q=record_q,
result_q=result_q,
interface=publish_interface,
)
ds = datastore.DataStore()
ds.open_for_scan(sync_item)
# save exit for final send
exit_pb = None
shown = False
while True:
data = ds.scan_data()
if data is None:
break
pb = wandb_internal_pb2.Record()
pb.ParseFromString(data)
record_type = pb.WhichOneof("record_type")
if self._view:
if self._verbose:
print("Record:", pb)
else:
print("Record:", record_type)
continue
if record_type == "run":
if self._run_id:
pb.run.run_id = self._run_id
if self._project:
pb.run.project = self._project
if self._entity:
pb.run.entity = self._entity
pb.control.req_resp = True
elif record_type == "exit":
exit_pb = pb
continue
elif record_type == "final":
assert exit_pb, "final seen without exit"
pb = exit_pb
exit_pb = None
sm.send(pb)
# send any records that were added in previous send
while not record_q.empty():
data = record_q.get(block=True)
sm.send(data)
if pb.control.req_resp:
result = result_q.get(block=True)
result_type = result.WhichOneof("result_type")
if not shown and result_type == "run_result":
r = result.run_result.run
# TODO(jhr): hardcode until we have settings in sync
url = "{}/{}/{}/runs/{}".format(
self._app_url,
url_quote(r.entity),
url_quote(r.project),
url_quote(r.run_id),
)
print("Syncing: %s ..." % url, end="")
sys.stdout.flush()
shown = True
sm.finish()
if self._mark_synced:
synced_file = "{}{}".format(sync_item, SYNCED_SUFFIX)
with open(synced_file, "w"):
pass
print("done.")