def headless(args):
"""Headless mode is where we start a monitoring / syncing
process to watch an already-running user process. It's like
`wandb run` for a user process that has already started.
The user process that calls this waits for a signal that
everything is ready, which is sent at the end of rm.wrap_existing_process
"""
user_process_pid = args['pid']
stdout_master_fd = args['stdout_master_fd']
stderr_master_fd = args['stderr_master_fd']
try:
run = wandb.wandb_run.Run.from_environment_or_defaults()
run.enable_logging()
# Clear inited for the wandb process
del os.environ[env.INITED]
# Set global run so save can work
wandb.run = run
# Access history to avoid sync issues
run.history
rm = wandb.run_manager.RunManager(run,
cloud=args['cloud'],
port=args['port'])
# We add a reference to _run_manager to enable wandb.save to work for tfevents files
# TODO: REFACTOR
run._run_manager = rm
rm.wrap_existing_process(user_process_pid, stdout_master_fd,
stderr_master_fd)
except Exception as e:
util.sentry_reraise(e)
def headless(args):
"""Headless mode is where we start a monitoring / syncing
process to watch an already-running user process. It's like
`wandb run` for a user process that has already started.
The user process that calls this waits for a signal that
everything is ready, which is sent at the end of rm.wrap_existing_process
"""
user_process_pid = args['pid']
stdout_master_fd = args['stdout_master_fd']
stderr_master_fd = args['stderr_master_fd']
try:
run = wandb.wandb_run.Run.from_environment_or_defaults()
run.enable_logging()
api = wandb.apis.InternalApi()
api.set_current_run_id(run.id)
rm = wandb.run_manager.RunManager(
api, run, cloud=args['cloud'],
port=args['port'])
rm.wrap_existing_process(
user_process_pid, stdout_master_fd, stderr_master_fd)
except Exception as e:
util.sentry_reraise(e)
def upload_file(self, url, file, callback=None, extra_headers={}):
"""Uploads a file to W&B with failure resumption
Args:
url (str): The url to download
file (str): The path to the file you want to upload
callback (:obj:`func`, optional): A callback which is passed the number of
bytes uploaded since the last time it was called, used to report progress
Returns:
The requests library response object
"""
extra_headers = extra_headers.copy()
response = None
progress = Progress(file, callback=callback)
if progress.len == 0:
raise CommError("%s is an empty file" % file.name)
try:
response = requests.put(
url, data=progress, headers=extra_headers)
response.raise_for_status()
except requests.exceptions.RequestException as e:
status_code = e.response.status_code if e.response != None else 0
# Retry errors from cloud storage or local network issues
if status_code in (308, 409, 429, 500, 502, 503, 504) or isinstance(e, (requests.exceptions.Timeout, requests.exceptions.ConnectionError)):
util.sentry_reraise(retry.TransientException(exc=e))
else:
util.sentry_reraise(e)
return response
def upload_file(self, url, file, callback=None, extra_headers={}):
"""Uploads a file to W&B with failure resumption
Args:
url (str): The url to download
file (str): The path to the file you want to upload
callback (:obj:`func`, optional): A callback which is passed the number of
bytes uploaded since the last time it was called, used to report progress
Returns:
The requests library response object
"""
extra_headers = extra_headers.copy()
response = None
if os.stat(file.name).st_size == 0:
raise CommError("%s is an empty file" % file.name)
try:
progress = Progress(file, callback=callback)
response = requests.put(url, data=progress, headers=extra_headers)
response.raise_for_status()
except requests.exceptions.RequestException as e:
total = progress.len
status = self._status_request(url, total)
# TODO(adrian): there's probably even more stuff we should add here
# like if we're offline, we should retry then too
if status.status_code in (308, 408, 500, 502, 503, 504):
util.sentry_reraise(retry.TransientException(exc=e))
else:
util.sentry_reraise(e)
return response
