def headless(args): """Headless mode is where we start a monitoring / syncing process to watch an already-running user process. It's like `wandb run` for a user process that has already started. The user process that calls this waits for a signal that everything is ready, which is sent at the end of rm.wrap_existing_process """ user_process_pid = args['pid'] stdout_master_fd = args['stdout_master_fd'] stderr_master_fd = args['stderr_master_fd'] try: run = wandb.wandb_run.Run.from_environment_or_defaults() run.enable_logging() # Clear inited for the wandb process del os.environ[env.INITED] # Set global run so save can work wandb.run = run # Access history to avoid sync issues run.history rm = wandb.run_manager.RunManager(run, cloud=args['cloud'], port=args['port']) # We add a reference to _run_manager to enable wandb.save to work for tfevents files # TODO: REFACTOR run._run_manager = rm rm.wrap_existing_process(user_process_pid, stdout_master_fd, stderr_master_fd) except Exception as e: util.sentry_reraise(e)
def headless(args): """Headless mode is where we start a monitoring / syncing process to watch an already-running user process. It's like `wandb run` for a user process that has already started. The user process that calls this waits for a signal that everything is ready, which is sent at the end of rm.wrap_existing_process """ user_process_pid = args['pid'] stdout_master_fd = args['stdout_master_fd'] stderr_master_fd = args['stderr_master_fd'] try: run = wandb.wandb_run.Run.from_environment_or_defaults() run.enable_logging() api = wandb.apis.InternalApi() api.set_current_run_id(run.id) rm = wandb.run_manager.RunManager( api, run, cloud=args['cloud'], port=args['port']) rm.wrap_existing_process( user_process_pid, stdout_master_fd, stderr_master_fd) except Exception as e: util.sentry_reraise(e)
def upload_file(self, url, file, callback=None, extra_headers={}): """Uploads a file to W&B with failure resumption Args: url (str): The url to download file (str): The path to the file you want to upload callback (:obj:`func`, optional): A callback which is passed the number of bytes uploaded since the last time it was called, used to report progress Returns: The requests library response object """ extra_headers = extra_headers.copy() response = None progress = Progress(file, callback=callback) if progress.len == 0: raise CommError("%s is an empty file" % file.name) try: response = requests.put( url, data=progress, headers=extra_headers) response.raise_for_status() except requests.exceptions.RequestException as e: status_code = e.response.status_code if e.response != None else 0 # Retry errors from cloud storage or local network issues if status_code in (308, 409, 429, 500, 502, 503, 504) or isinstance(e, (requests.exceptions.Timeout, requests.exceptions.ConnectionError)): util.sentry_reraise(retry.TransientException(exc=e)) else: util.sentry_reraise(e) return response
def upload_file(self, url, file, callback=None, extra_headers={}): """Uploads a file to W&B with failure resumption Args: url (str): The url to download file (str): The path to the file you want to upload callback (:obj:`func`, optional): A callback which is passed the number of bytes uploaded since the last time it was called, used to report progress Returns: The requests library response object """ extra_headers = extra_headers.copy() response = None if os.stat(file.name).st_size == 0: raise CommError("%s is an empty file" % file.name) try: progress = Progress(file, callback=callback) response = requests.put(url, data=progress, headers=extra_headers) response.raise_for_status() except requests.exceptions.RequestException as e: total = progress.len status = self._status_request(url, total) # TODO(adrian): there's probably even more stuff we should add here # like if we're offline, we should retry then too if status.status_code in (308, 408, 500, 502, 503, 504): util.sentry_reraise(retry.TransientException(exc=e)) else: util.sentry_reraise(e) return response