def normalize_job_name(raw_job_name, default_username='', default_project_name='', use_config=True): raw_job_name = raw_job_name or '' if use_config: default_project_name = default_project_name or current_project_name() name_parts = raw_job_name.split('/') namespace = default_username or current_project_namespace() project_name = default_project_name number = '' # current job number # When nothing is passed, use all the defaults if not raw_job_name: pass elif len(name_parts) == 4: # mckay/projects/foo/1 namespace, _, project_name, number = name_parts elif len(name_parts) == 3: if name_parts[2].isdigit(): # mckay/foo/1 namespace, project_name, number = name_parts else: # mckay/projects/foo namespace, _, project_name = name_parts elif len(name_parts) == 2: if name_parts[1].isdigit(): # foo/1 project_name, number = name_parts else: # mckay/foo namespace, project_name = name_parts elif len(name_parts) == 1: if name_parts[0].isdigit(): # 1 number = name_parts[0] else: # foo project_name = name_parts[0] else: return raw_job_name # If no number is found, query the API for the most recent job number if not number: job_name_from_api = get_latest_job_name_for_project(namespace, project_name) if not job_name_from_api: raise FloydException("Could not resolve %s. Make sure the project exists and has jobs." % raw_job_name) return job_name_from_api if not project_name: raise FloydException('Job name resolution: Could not infer a project name from "%s". Please include a name to identify the project' % raw_job_name) return '/'.join([namespace, 'projects', project_name, number])
def check_response_status(self, response): """ Check if response is successful. Else raise Exception. """ if not (200 <= response.status_code < 300): try: message = response.json()["errors"] except Exception: message = None floyd_logger.debug("Error received : status_code: {}, message: {}".format(response.status_code, message or response.content)) if response.status_code == 400: raise BadRequestException(response) elif response.status_code == 401: raise AuthenticationException() elif response.status_code == 403: raise AuthorizationException() elif response.status_code == 404: raise NotFoundException() elif response.status_code == 429: raise OverLimitException(response.json().get("message")) elif response.status_code == 502: raise BadGatewayException() elif response.status_code == 504: raise GatewayTimeoutException() elif 500 <= response.status_code < 600: if 'Server under maintenance' in response.content.decode(): raise ServerException('Server under maintenance, please try again later.') else: raise ServerException() else: msg = "An error occurred. Server response: {}".format(response.status_code) raise FloydException(message=msg)
def normalize_data_name(raw_name, default_username='', default_dataset_name='', use_data_config=True): raw_name = raw_name or '' if use_data_config: default_dataset_name = default_dataset_name or current_dataset_name() if raw_name.endswith('/output'): return normalize_job_name(raw_name[:-len('/output')], default_username, default_dataset_name) + '/output' name_parts = raw_name.split('/') username = default_username or current_username() name = default_dataset_name number = None # current version number # When nothing is passed, use all the defaults if not raw_name: pass elif len(name_parts) == 4: # mckay/datasets/foo/1 username, _, name, number = name_parts elif len(name_parts) == 3: if name_parts[2].isdigit(): # mckay/foo/1 username, name, number = name_parts else: # mckay/projects/foo username, _, name = name_parts elif len(name_parts) == 2: if name_parts[1].isdigit(): # foo/1 name, number = name_parts else: # mckay/foo username, name = name_parts elif len(name_parts) == 1: if name_parts[0].isdigit(): # 1 number = name_parts[0] else: # foo name = name_parts[0] else: return raw_name name_parts = [username, 'datasets', name] if number is not None: name_parts.append(number) if not name: raise FloydException( 'Dataset name resolution: Could not infer a name from "%s". Please include a name to identify the dataset' % raw_name) return '/'.join(name_parts)
def get_config(cls): if not os.path.isfile(cls.CONFIG_FILE_PATH): raise FloydException( "Missing .floydexpt file, run floyd init first") with open(cls.CONFIG_FILE_PATH, "r") as config_file: experiment_config_str = config_file.read() return ExperimentConfig.from_dict(json.loads(experiment_config_str))
def get_config(cls): if not os.path.isfile(cls.CONFIG_FILE_PATH): raise FloydException( "Missing .floyddata file, run floyd data init first") with open(cls.CONFIG_FILE_PATH, "r") as config_file: data_config_str = config_file.read() return DataConfig.from_dict(json.loads(data_config_str))
def get_url_contents(url): """ Downloads the content of the url and returns it """ response = requests.get(url) if response.status_code == 200: return response.content.decode(response.encoding) else: raise FloydException( "Failed to get contents of the url : {}".format(url))
def init(dataset_name): """ Initialize a new dataset at the current dir. Then run the upload command to copy all the files in this directory to FloydHub. floyd data upload """ dataset_obj = DatasetClient().get_by_name(dataset_name) if not dataset_obj: namespace, name = get_namespace_from_name(dataset_name) create_dataset_base_url = "{}/datasets/create".format( floyd.floyd_web_host) create_dataset_url = "{}?name={}&namespace={}".format( create_dataset_base_url, name, namespace) floyd_logger.info( ("Dataset name does not match your list of datasets. " "Create your new dataset in the web dashboard:\n\t%s"), create_dataset_base_url) webbrowser.open(create_dataset_url) name = click.prompt( 'Press ENTER to use dataset name "%s" or enter a different name' % dataset_name, default=dataset_name, show_default=False) dataset_name = name.strip() or dataset_name dataset_obj = DatasetClient().get_by_name(dataset_name) if not dataset_obj: raise FloydException( 'Dataset "%s" does not exist on floydhub.com. Ensure it exists before continuing.' % dataset_name) namespace, name = get_namespace_from_name(dataset_name) data_config = DataConfig(name=name, namespace=namespace, family_id=dataset_obj.id) DataConfigManager.set_config(data_config) floyd_logger.info( "Data source \"{}\" initialized in current directory".format( dataset_name)) floyd_logger.info(""" You can now upload your data to Floyd by: floyd data upload """)
def init(project_name): """ Initialize new project at the current dir. After init run your command. Example: floyd run 'python tensorflow.py > /output/model.1' """ project_obj = ProjectClient().get_by_name(project_name) if not project_obj: namespace, name = get_namespace_from_name(project_name) create_project_base_url = "{}/projects/create".format( floyd.floyd_web_host) create_project_url = "{}?name={}&namespace={}".format( create_project_base_url, name, namespace) floyd_logger.info(('Project name does not yet exist on floydhub.com. ' 'Create your new project on floydhub.com:\n\t%s'), create_project_base_url) webbrowser.open(create_project_url) name = click.prompt( 'Press ENTER to use project name "%s" or enter a different name' % project_name, default=project_name, show_default=False) project_name = name.strip() or project_name project_obj = ProjectClient().get_by_name(project_name) if not project_obj: raise FloydException( 'Project "%s" does not exist on floydhub.com. Ensure it exists before continuing.' % project_name) namespace, name = get_namespace_from_name(project_name) experiment_config = ExperimentConfig(name=name, namespace=namespace, family_id=project_obj.id) ExperimentConfigManager.set_config(experiment_config) FloydIgnoreManager.init() yaml_config = read_yaml_config() if not yaml_config: copyfile(os.path.join(os.path.dirname(__file__), 'default_floyd.yml'), 'floyd.yml') floyd_logger.info("Project \"%s\" initialized in current directory", project_name)
def check_cli_version(): """ Check if the current cli version satisfies the server requirements """ server_version = VersionClient().get_cli_version() current_version = pkg_resources.require("floyd-cli")[0].version if LooseVersion(current_version) < LooseVersion( server_version.min_version): raise FloydException(""" Your version of CLI ({}) is no longer compatible with server. Run: pip install -U floyd-cli to upgrade to the latest version ({}) """.format(current_version, server_version.latest_version)) if LooseVersion(current_version) < LooseVersion( server_version.latest_version): print(""" New version of CLI ({}) is now available. To upgrade run: pip install -U floyd-cli """.format(server_version.latest_version))