def normalize_job_name(raw_job_name, default_username='', default_project_name='', use_config=True):
raw_job_name = raw_job_name or ''
if use_config:
default_project_name = default_project_name or current_project_name()
name_parts = raw_job_name.split('/')
namespace = default_username or current_project_namespace()
project_name = default_project_name
number = '' # current job number
# When nothing is passed, use all the defaults
if not raw_job_name:
pass
elif len(name_parts) == 4:
# mckay/projects/foo/1
namespace, _, project_name, number = name_parts
elif len(name_parts) == 3:
if name_parts[2].isdigit():
# mckay/foo/1
namespace, project_name, number = name_parts
else:
# mckay/projects/foo
namespace, _, project_name = name_parts
elif len(name_parts) == 2:
if name_parts[1].isdigit():
# foo/1
project_name, number = name_parts
else:
# mckay/foo
namespace, project_name = name_parts
elif len(name_parts) == 1:
if name_parts[0].isdigit():
# 1
number = name_parts[0]
else:
# foo
project_name = name_parts[0]
else:
return raw_job_name
# If no number is found, query the API for the most recent job number
if not number:
job_name_from_api = get_latest_job_name_for_project(namespace, project_name)
if not job_name_from_api:
raise FloydException("Could not resolve %s. Make sure the project exists and has jobs." % raw_job_name)
return job_name_from_api
if not project_name:
raise FloydException('Job name resolution: Could not infer a project name from "%s". Please include a name to identify the project' % raw_job_name)
return '/'.join([namespace, 'projects', project_name, number])
def check_response_status(self, response):
"""
Check if response is successful. Else raise Exception.
"""
if not (200 <= response.status_code < 300):
try:
message = response.json()["errors"]
except Exception:
message = None
floyd_logger.debug("Error received : status_code: {}, message: {}".format(response.status_code,
message or response.content))
if response.status_code == 400:
raise BadRequestException(response)
elif response.status_code == 401:
raise AuthenticationException()
elif response.status_code == 403:
raise AuthorizationException()
elif response.status_code == 404:
raise NotFoundException()
elif response.status_code == 429:
raise OverLimitException(response.json().get("message"))
elif response.status_code == 502:
raise BadGatewayException()
elif response.status_code == 504:
raise GatewayTimeoutException()
elif 500 <= response.status_code < 600:
if 'Server under maintenance' in response.content.decode():
raise ServerException('Server under maintenance, please try again later.')
else:
raise ServerException()
else:
msg = "An error occurred. Server response: {}".format(response.status_code)
raise FloydException(message=msg)
def normalize_data_name(raw_name,
default_username='',
default_dataset_name='',
use_data_config=True):
raw_name = raw_name or ''
if use_data_config:
default_dataset_name = default_dataset_name or current_dataset_name()
if raw_name.endswith('/output'):
return normalize_job_name(raw_name[:-len('/output')], default_username,
default_dataset_name) + '/output'
name_parts = raw_name.split('/')
username = default_username or current_username()
name = default_dataset_name
number = None # current version number
# When nothing is passed, use all the defaults
if not raw_name:
pass
elif len(name_parts) == 4:
# mckay/datasets/foo/1
username, _, name, number = name_parts
elif len(name_parts) == 3:
if name_parts[2].isdigit():
# mckay/foo/1
username, name, number = name_parts
else:
# mckay/projects/foo
username, _, name = name_parts
elif len(name_parts) == 2:
if name_parts[1].isdigit():
# foo/1
name, number = name_parts
else:
# mckay/foo
username, name = name_parts
elif len(name_parts) == 1:
if name_parts[0].isdigit():
# 1
number = name_parts[0]
else:
# foo
name = name_parts[0]
else:
return raw_name
name_parts = [username, 'datasets', name]
if number is not None:
name_parts.append(number)
if not name:
raise FloydException(
'Dataset name resolution: Could not infer a name from "%s". Please include a name to identify the dataset'
% raw_name)
return '/'.join(name_parts)
def get_config(cls):
if not os.path.isfile(cls.CONFIG_FILE_PATH):
raise FloydException(
"Missing .floydexpt file, run floyd init first")
with open(cls.CONFIG_FILE_PATH, "r") as config_file:
experiment_config_str = config_file.read()
return ExperimentConfig.from_dict(json.loads(experiment_config_str))
def get_config(cls):
if not os.path.isfile(cls.CONFIG_FILE_PATH):
raise FloydException(
"Missing .floyddata file, run floyd data init first")
with open(cls.CONFIG_FILE_PATH, "r") as config_file:
data_config_str = config_file.read()
return DataConfig.from_dict(json.loads(data_config_str))
def get_url_contents(url):
"""
Downloads the content of the url and returns it
"""
response = requests.get(url)
if response.status_code == 200:
return response.content.decode(response.encoding)
else:
raise FloydException(
"Failed to get contents of the url : {}".format(url))
def init(dataset_name):
"""
Initialize a new dataset at the current dir.
Then run the upload command to copy all the files in this
directory to FloydHub.
floyd data upload
"""
dataset_obj = DatasetClient().get_by_name(dataset_name)
if not dataset_obj:
namespace, name = get_namespace_from_name(dataset_name)
create_dataset_base_url = "{}/datasets/create".format(
floyd.floyd_web_host)
create_dataset_url = "{}?name={}&namespace={}".format(
create_dataset_base_url, name, namespace)
floyd_logger.info(
("Dataset name does not match your list of datasets. "
"Create your new dataset in the web dashboard:\n\t%s"),
create_dataset_base_url)
webbrowser.open(create_dataset_url)
name = click.prompt(
'Press ENTER to use dataset name "%s" or enter a different name' %
dataset_name,
default=dataset_name,
show_default=False)
dataset_name = name.strip() or dataset_name
dataset_obj = DatasetClient().get_by_name(dataset_name)
if not dataset_obj:
raise FloydException(
'Dataset "%s" does not exist on floydhub.com. Ensure it exists before continuing.'
% dataset_name)
namespace, name = get_namespace_from_name(dataset_name)
data_config = DataConfig(name=name,
namespace=namespace,
family_id=dataset_obj.id)
DataConfigManager.set_config(data_config)
floyd_logger.info(
"Data source \"{}\" initialized in current directory".format(
dataset_name))
floyd_logger.info("""
You can now upload your data to Floyd by:
floyd data upload
""")
def init(project_name):
"""
Initialize new project at the current dir.
After init run your command. Example:
floyd run 'python tensorflow.py > /output/model.1'
"""
project_obj = ProjectClient().get_by_name(project_name)
if not project_obj:
namespace, name = get_namespace_from_name(project_name)
create_project_base_url = "{}/projects/create".format(
floyd.floyd_web_host)
create_project_url = "{}?name={}&namespace={}".format(
create_project_base_url, name, namespace)
floyd_logger.info(('Project name does not yet exist on floydhub.com. '
'Create your new project on floydhub.com:\n\t%s'),
create_project_base_url)
webbrowser.open(create_project_url)
name = click.prompt(
'Press ENTER to use project name "%s" or enter a different name' %
project_name,
default=project_name,
show_default=False)
project_name = name.strip() or project_name
project_obj = ProjectClient().get_by_name(project_name)
if not project_obj:
raise FloydException(
'Project "%s" does not exist on floydhub.com. Ensure it exists before continuing.'
% project_name)
namespace, name = get_namespace_from_name(project_name)
experiment_config = ExperimentConfig(name=name,
namespace=namespace,
family_id=project_obj.id)
ExperimentConfigManager.set_config(experiment_config)
FloydIgnoreManager.init()
yaml_config = read_yaml_config()
if not yaml_config:
copyfile(os.path.join(os.path.dirname(__file__), 'default_floyd.yml'),
'floyd.yml')
floyd_logger.info("Project \"%s\" initialized in current directory",
project_name)
def check_cli_version():
"""
Check if the current cli version satisfies the server requirements
"""
server_version = VersionClient().get_cli_version()
current_version = pkg_resources.require("floyd-cli")[0].version
if LooseVersion(current_version) < LooseVersion(
server_version.min_version):
raise FloydException("""
Your version of CLI ({}) is no longer compatible with server. Run:
pip install -U floyd-cli
to upgrade to the latest version ({})
""".format(current_version, server_version.latest_version))
if LooseVersion(current_version) < LooseVersion(
server_version.latest_version):
print("""
New version of CLI ({}) is now available. To upgrade run:
pip install -U floyd-cli
""".format(server_version.latest_version))
