def create_temp(prefix, template):
"""Create workspace templates."""
echo('Creating Workspace Templates')
copytree(src=WORKSPACE / template,
dst=prefix,
dirs_exist_ok=True,
ignore=ignore_patterns('__pycache__')) # from template workspace
def export_():
"""Export federated learning workspace."""
from shutil import make_archive, copytree, copy2, ignore_patterns
from tempfile import mkdtemp
from os import getcwd, makedirs
from os.path import basename, join
from plan import FreezePlan
# TODO: Does this need to freeze all plans?
planFile = 'plan/plan.yaml'
try:
FreezePlan(planFile)
except Exception:
echo(f'Plan file "{planFile}" not found. No freeze performed.')
from pip._internal.operations import freeze
requirements_generator = freeze.freeze()
with open('./requirements.txt', 'w') as f:
for package in requirements_generator:
if '==' not in package:
# We do not export dependencies without version
continue
f.write(package + '\n')
archiveType = 'zip'
archiveName = basename(getcwd())
archiveFileName = archiveName + '.' + archiveType
# Aggregator workspace
tmpDir = join(mkdtemp(), 'openfl', archiveName)
ignore = ignore_patterns('__pycache__', '*.crt', '*.key', '*.csr', '*.srl',
'*.pem', '*.pbuf')
# We only export the minimum required files to set up a collaborator
makedirs(f'{tmpDir}/save', exist_ok=True)
makedirs(f'{tmpDir}/logs', exist_ok=True)
makedirs(f'{tmpDir}/data', exist_ok=True)
copytree('./code', f'{tmpDir}/code', ignore=ignore) # code
copytree('./plan', f'{tmpDir}/plan', ignore=ignore) # plan
copy2('./requirements.txt', f'{tmpDir}/requirements.txt') # requirements
try:
copy2('.workspace', tmpDir) # .workspace
except FileNotFoundError:
echo('\'.workspace\' file not found.')
if confirm('Create a default \'.workspace\' file?'):
copy2(WORKSPACE / 'workspace' / '.workspace', tmpDir)
else:
echo('To proceed, you must have a \'.workspace\' '
'file in the current directory.')
raise
# Create Zip archive of directory
make_archive(archiveName, archiveType, tmpDir)
echo(f'Workspace exported to archive: {archiveFileName}')
def create_dirs(prefix):
"""Create workspace directories."""
echo('Creating Workspace Directories')
(prefix / 'cert').mkdir(parents=True, exist_ok=True) # certifications
(prefix / 'data').mkdir(parents=True, exist_ok=True) # training data
(prefix / 'logs').mkdir(parents=True, exist_ok=True) # training logs
(prefix / 'plan').mkdir(parents=True,
exist_ok=True) # federated learning plans
(prefix / 'save').mkdir(
parents=True, exist_ok=True) # model weight saves / initialization
(prefix / 'code').mkdir(parents=True, exist_ok=True) # model code
src = WORKSPACE / 'workspace/plan/defaults' # from default workspace
dst = prefix / 'plan/defaults' # to created workspace
copytree(src=src, dst=dst, dirs_exist_ok=True)
copyfile(WORKSPACE / 'workspace' / '.workspace', prefix / '.workspace')
def export_():
"""Export federated learning workspace."""
from shutil import make_archive, copytree, copy2, ignore_patterns
from tempfile import mkdtemp
from os import getcwd, makedirs
from os.path import basename, join
from plan import FreezePlan
# TODO: Does this need to freeze all plans?
planFile = 'plan/plan.yaml'
try:
FreezePlan(planFile)
except Exception:
echo(f'Plan file "{planFile}" not found. No freeze performed.')
requirements_filename = 'requirements.txt'
workspace_reqs = _get_requirements_dict(requirements_filename)
prefix = getcwd()
export_requirements_filename = 'requirements.export.txt'
with open(export_requirements_filename, "w") as f:
check_call([executable, "-m", "pip", "freeze"], shell=False, stdout=f)
workspace_hash = _get_dir_hash(prefix)
origin_dict = _get_requirements_dict(OPENFL_USERDIR /
f'requirements.{workspace_hash}.txt')
current_dict = _get_requirements_dict(export_requirements_filename)
with open(export_requirements_filename, "w") as f:
for package, version in current_dict.items():
if (package in workspace_reqs or package not in origin_dict
or version != origin_dict[package]):
# we save only the difference between original workspace after
# 'fx create workspace' and current one.
echo(f'Writing {package}=={version} '
f'to {requirements_filename}...')
f.write(f'{package}=={version}\n')
elif version is None: # local dependency
warn(f'Could not generate requirements for {package}.'
f' Consider installing it manually after workspace'
f' import.')
echo(f'{export_requirements_filename} written.')
archiveType = 'zip'
archiveName = basename(getcwd())
archiveFileName = archiveName + '.' + archiveType
# Aggregator workspace
tmpDir = join(mkdtemp(), 'openfl', archiveName)
ignore = ignore_patterns('__pycache__', '*.crt', '*.key', '*.csr', '*.srl',
'*.pem', '*.pbuf')
# We only export the minimum required files to set up a collaborator
makedirs(f'{tmpDir}/save', exist_ok=True)
makedirs(f'{tmpDir}/logs', exist_ok=True)
makedirs(f'{tmpDir}/data', exist_ok=True)
copytree('./code', f'{tmpDir}/code', ignore=ignore) # code
copytree('./plan', f'{tmpDir}/plan', ignore=ignore) # plan
copy2(export_requirements_filename,
f'{tmpDir}/requirements.txt') # requirements
try:
copy2('.workspace', tmpDir) # .workspace
except FileNotFoundError:
echo('\'.workspace\' file not found.')
if confirm('Create a default \'.workspace\' file?'):
copy2(WORKSPACE / 'workspace' / '.workspace', tmpDir)
else:
echo('To proceed, you must have a \'.workspace\' '
'file in the current directory.')
raise
# Create Zip archive of directory
make_archive(archiveName, archiveType, tmpDir)
echo(f'Workspace exported to archive: {archiveFileName}')