def init(project):
"""
Initialize new project at the current dir.
After init run your command. Example:
floyd run 'python tensorflow.py > /output/model.1'
"""
project_obj = ProjectClient().get_by_name(project)
if not project_obj:
create_project_base_url = "{}/projects/create".format(
floyd.floyd_web_host)
create_project_url = "{}?name={}".format(create_project_base_url,
project)
floyd_logger.error(
("Project name does not match your list of projects. "
"Create your new project in the web dashboard:\n\t%s"),
create_project_base_url)
webbrowser.open(create_project_url)
return
experiment_config = ExperimentConfig(name=project,
family_id=project_obj.id)
ExperimentConfigManager.set_config(experiment_config)
FloydIgnoreManager.init()
floyd_logger.info(
"Project \"{}\" initialized in current directory".format(project))
def init(project):
"""
Initialize new project at the current dir.
After init run your command. Example:
floyd run python tensorflow.py > /output/model.1
"""
experiment_config = ExperimentConfig(name=project,
family_id=generate_uuid())
ExperimentConfigManager.set_config(experiment_config)
FloydIgnoreManager.init()
floyd_logger.info(
"Project \"{}\" initialized in current directory".format(project))
def init(project_name):
"""
Initialize new project at the current dir.
After init run your command. Example:
floyd run 'python tensorflow.py > /output/model.1'
"""
project_obj = ProjectClient().get_by_name(project_name)
if not project_obj:
namespace, name = get_namespace_from_name(project_name)
create_project_base_url = "{}/projects/create".format(
floyd.floyd_web_host)
create_project_url = "{}?name={}&namespace={}".format(
create_project_base_url, name, namespace)
floyd_logger.info(('Project name does not yet exist on floydhub.com. '
'Create your new project on floydhub.com:\n\t%s'),
create_project_base_url)
webbrowser.open(create_project_url)
name = click.prompt(
'Press ENTER to use project name "%s" or enter a different name' %
project_name,
default=project_name,
show_default=False)
project_name = name.strip() or project_name
project_obj = ProjectClient().get_by_name(project_name)
if not project_obj:
raise FloydException(
'Project "%s" does not exist on floydhub.com. Ensure it exists before continuing.'
% project_name)
namespace, name = get_namespace_from_name(project_name)
experiment_config = ExperimentConfig(name=name,
namespace=namespace,
family_id=project_obj.id)
ExperimentConfigManager.set_config(experiment_config)
FloydIgnoreManager.init()
yaml_config = read_yaml_config()
if not yaml_config:
copyfile(os.path.join(os.path.dirname(__file__), 'default_floyd.yml'),
'floyd.yml')
floyd_logger.info("Project \"%s\" initialized in current directory",
project_name)
def get_files_in_directory(path, file_type):
"""
Gets the list of files in the directory and subdirectories
Respects .floydignore file if present
"""
local_files = []
separator = os.path.sep
ignore_list = FloydIgnoreManager.get_list()
ignore_list_localized = [
".{}{}".format(separator, item) for item in ignore_list
]
floyd_logger.debug("Ignoring list : {}".format(ignore_list_localized))
for root, dirs, files in os.walk(path):
ignore_dir = False
for item in ignore_list_localized:
if root.startswith(item):
ignore_dir = True
if ignore_dir:
floyd_logger.debug("Ignoring directory : {}".format(root))
continue
for file_name in files:
file_relative_path = os.path.join(root, file_name)
if separator != '/': # convert relative paths to Unix style
file_relative_path = file_relative_path.replace(
os.path.sep, '/')
file_full_path = os.path.join(os.getcwd(), root, file_name)
local_files.append(
(file_type, (file_relative_path, open(file_full_path,
'rb'), 'text/plain')))
return local_files
def test_escaping_of_globs_that_start_with_reserved_chars(self, mopen, _):
# Manually mock the iterator that imitates the lines read from the file
file_data = ['', '# comment', '\#file_name', '\!file_name']
mopen.return_value.__iter__.return_value = file_data
result = FloydIgnoreManager.get_lists()
self.assertEqual(result, (['#file_name', '!file_name'], []))
def test_properly_interprets_whitelisted_globs(self, mopen, _):
# Manually mock the iterator that imitates the lines read from the file
file_data = ['', '# comment', '*.py', '!hello.py']
mopen.return_value.__iter__.return_value = file_data
result = FloydIgnoreManager.get_lists()
self.assertEqual(result, (['*.py'], ['hello.py']))
def test_trims_slash_prefix_from_abs_paths(self, mopen, _):
# Manually mock the iterator that imitates the lines read from the file
file_data = ['/hello', '!/world']
mopen.return_value.__iter__.return_value = file_data
result = FloydIgnoreManager.get_lists()
self.assertEqual(result, (['hello'], ['world']))
def test_ignores_commented_lines(self, mopen, _):
# Manually mock the iterator that imitates the lines read from the file
file_data = ['', '# comment', '', '*.py']
mopen.return_value.__iter__.return_value = file_data
result = FloydIgnoreManager.get_lists()
self.assertEqual(result, (['*.py'], []))
def get_files_in_directory(path, file_type):
"""
Gets the list of files in the directory and subdirectories
Respects .floydignore file if present
"""
local_files = []
separator = os.path.sep
ignore_list = FloydIgnoreManager.get_list()
# make sure that subdirectories are also excluded
ignore_list_expanded = ignore_list + ["{}/**".format(item) for item in ignore_list]
floyd_logger.debug("Ignoring list : {}".format(ignore_list))
total_file_size = 0
for root, dirs, files in os.walk(path):
dirs[:] = [d for d in dirs if d not in ignore_list]
ignore_dir = False
normalized_path = normalize_path(path, root)
for item in ignore_list_expanded:
if PurePath(normalized_path).match(item):
ignore_dir = True
break
if ignore_dir:
floyd_logger.debug("Ignoring directory : {}".format(root))
continue
for file_name in files:
ignore_file = False
normalized_path = normalize_path(path, os.path.join(root, file_name))
for item in ignore_list_expanded:
if PurePath(normalized_path).match(item):
ignore_file = True
break
if ignore_file:
floyd_logger.debug("Ignoring file : {}".format(normalized_path))
continue
file_relative_path = os.path.join(root, file_name)
if separator != '/': # convert relative paths to Unix style
file_relative_path = file_relative_path.replace(os.path.sep, '/')
file_full_path = os.path.join(os.getcwd(), root, file_name)
local_files.append((file_type, (file_relative_path, open(file_full_path, 'rb'), 'text/plain')))
total_file_size += os.path.getsize(file_full_path)
return (local_files, sizeof_fmt(total_file_size))
def get_files_in_current_directory(file_type):
"""
Gets the list of files in the current directory and subdirectories.
Respects .floydignore file if present
"""
local_files = []
total_file_size = 0
ignore_list, whitelist = FloydIgnoreManager.get_lists()
floyd_logger.debug("Ignoring: %s", ignore_list)
floyd_logger.debug("Whitelisting: %s", whitelist)
file_paths = get_unignored_file_paths(ignore_list, whitelist)
for file_path in file_paths:
local_files.append((file_type, (unix_style_path(file_path), open(file_path, 'rb'), 'text/plain')))
total_file_size += os.path.getsize(file_path)
return (local_files, sizeof_fmt(total_file_size))
def test_returns_two_empty_lists_if_file_is_not_present(self, _):
result = FloydIgnoreManager.get_lists()
self.assertEqual(result, ([], []))