def test_paths_to_copy(self):
with tempdir() as tmp:
tmp.chdir()
(tmp / 'a/1').makedirs()
(tmp / 'a/2').makedirs()
(tmp / 'b/1').makedirs()
(tmp / 'b/2').makedirs()
(tmp / 'c/1/a').makedirs()
(tmp / 'file1').write_text('file1')
(tmp / 'file2').write_text('file2')
(tmp / 'file3').write_text('file3')
(tmp / 'a/file_a1').write_text('file_a1')
(tmp / 'a/1/file_a1_1').write_text('file_a1_1')
# list all dirs and files
self.assertEqual(
{
PathToDump(Path(p), Path(p))
for p in {'a', 'b', 'c', 'file1', 'file2', 'file3'}
}, set(get_paths_to_copy()))
# exclude 'a' and 'file1'
self.assertEqual(
{
PathToDump(Path(p), Path(p))
for p in {'b', 'c', 'file2', 'file3'}
}, set(get_paths_to_copy(exclude=['a', 'file1'])))
# exclude 'a/1 and 'file1'
self.assertEqual(
{
PathToDump(Path(p), Path(p))
for p in {'a/2', 'a/file_a1', 'b', 'c', 'file2', 'file3'}
},
set(
get_paths_to_copy(
exclude=['a/1', 'file1', 'a/10/file_a10_1'])))
# add external resource
self.assertEqual(
{
PathToDump(Path(s), Path(d))
for s, d in {('../external1', 'external1'), (
'a', 'a'), ('b', 'b'), ('c', 'c'), (
'file1', 'file1'), ('file2',
'file2'), ('file3', 'file3')}
},
set(get_paths_to_copy(paths_to_copy=[tmp / '../external1'])))
def deploy_code(self, experiment):
paths_to_dump = get_paths_to_copy(
exclude=experiment.exclude, paths_to_copy=experiment.paths_to_copy)
with tempfile.NamedTemporaryFile(suffix='.tar.gz') as temp_file:
# archive all files
with tarfile.open(temp_file.name, 'w:gz') as tar_file:
for p in paths_to_dump:
LOGGER.debug('Adding "{}" to deployment archive'.format(
p.rel_remote_path))
tar_file.add(p.local_path, arcname=p.rel_remote_path)
if experiment.neptune_token_files:
neptune_token_path = experiment.neptune_token_files[0]
rel_local_path = Path('.').relpathto(neptune_token_path)
remote_path = '/'.join([
p for p in rel_local_path.split('/') if p and p != '..'
])
LOGGER.debug('Adding "{}" to deployment archive'.format(
remote_path))
tar_file.add(neptune_token_path, arcname=remote_path)
# upload archive to cluster and extract
self._put(temp_file.name, experiment.experiment_scratch_dir)
with cd(experiment.experiment_scratch_dir):
archive_remote_path = experiment.experiment_scratch_dir / Path(
temp_file.name).name
self._fabric_run(
'tar xf {tar_filename} && rm {tar_filename}'.format(
tar_filename=archive_remote_path))
# create and upload experiment script
script = ExperimentScript(experiment)
remote_script_path = experiment.project_scratch_dir / script.script_name
self._put(script.path, remote_script_path)
return remote_script_path
def __init__(self, experiment, requirements_file):
experiment_data = attr.asdict(experiment)
# paths in command shall be relative
cmd = experiment_data.pop('cmd')
updated_cmd = self._rewrite_paths(experiment.cwd, cmd.command)
paths_to_copy = get_paths_to_copy(
exclude=experiment.exclude, paths_to_copy=experiment.paths_to_copy)
experiment = attr.evolve(experiment,
cmd=StaticCmd(command=updated_cmd,
env=cmd.env))
super(DockerFile, self).__init__(
template_filename=self.DEFAULT_DOCKERFILE_TEMPLATE,
experiment=experiment,
requirements_file=requirements_file,
paths_to_copy=paths_to_copy)