def test_bundle_job_adds_config_files(self):
mock_job_source_bundle, mock_tar = self._setup_archive_and_tar()
self.mock_glob_glob.return_value = ['fake_config_filename.config.yaml']
job_bundler = JobBundler('fake_name', {}, None, mock_job_source_bundle)
job_bundler._bundle_job()
add_config_call = call('fake_config_filename.config.yaml', arcname='fake_name/fake_config_filename.config.yaml')
mock_tar.add.assert_has_calls([add_config_call])
def test_cleanup_removes_correct_files(self):
job_bundler = JobBundler('fake_name', {}, None, None)
job_bundler.cleanup()
remove_archive = call(self.temp_directory + '/fake_name.tgz')
remove_bin = call('fake_name.bin')
remove_config = call(self.temp_directory + '/fake_name.config.yaml')
self.mock_os_remove.assert_has_calls([remove_archive, remove_bin, remove_config])
def test_save_config_dumps_to_file(self):
mock_job = self._create_mock_job()
return_object = self.MockContextManager()
self.mock_builtins_open.return_value = return_object
job_bundler = JobBundler('fake_name', {}, mock_job, None)
job_bundler._save_config()
self.mock_yaml_dump.assert_called_with({'job_name': 'fake_name'}, return_object)
def test_save_job_writes_to_file(self):
mock_job = self._create_mock_job()
return_object = self.MockContextManager()
self.mock_builtins_open.return_value = return_object
job_bundler = JobBundler('fake_name', {}, mock_job, None)
job_bundler._save_job()
return_object.write.assert_called_with('something')
def __init__(self, job_name, job, job_source_bundle, code_bucket, results_bucket):
from foundations_contrib.global_state import config_manager
from foundations_contrib.job_bundler import JobBundler
self._config = {}
self._config.update(config_manager.config())
self._config['_is_deployment'] = True
self._code_bucket = code_bucket
self._result_bucket = results_bucket
self._job_name = job_name
self._job = job
self._job_bundler = JobBundler(
self._job_name, self._config, self._job, job_source_bundle)
self._job_results = None
def job_bundle(bundle_name):
from foundations_contrib.job_bundler import JobBundler
from foundations_contrib.job_bundling.empty_job import EmptyJob
from foundations_contrib.job_bundling.folder_job_source_bundle import FolderJobSourceBundle
from foundations_contrib.global_state import config_manager
return JobBundler(bundle_name, config_manager.config(), EmptyJob(),
FolderJobSourceBundle(), 'job_source')
class BucketJobDeployment(object):
def __init__(self, job_name, job, job_source_bundle, code_bucket, results_bucket):
from foundations_contrib.global_state import config_manager
from foundations_contrib.job_bundler import JobBundler
self._config = {}
self._config.update(config_manager.config())
self._config['_is_deployment'] = True
self._code_bucket = code_bucket
self._result_bucket = results_bucket
self._job_name = job_name
self._job = job
self._job_bundler = JobBundler(
self._job_name, self._config, self._job, job_source_bundle)
self._job_results = None
def config(self):
return self._config
def job_name(self):
return self._job_name
def deploy(self):
self._job_bundler.bundle()
try:
self._run()
finally:
self._job_bundler.cleanup()
def is_job_complete(self):
return self._result_bucket.exists(self._job_archive_name())
def fetch_job_results(self):
import os
import tarfile
from foundations_internal.serializer import deserialize_from_file
from foundations_contrib.simple_tempfile import SimpleTempfile
if self._job_results is None:
with SimpleTempfile('w+b') as temp_file:
self._result_bucket.download_to_file(
self._job_archive_name(), temp_file)
with tarfile.open(temp_file.name, "r:gz") as tar:
for tarinfo in tar:
if os.path.splitext(tarinfo.name)[1] == ".pkl":
file = tar.extractfile(tarinfo)
self._job_results = deserialize_from_file(file)
file.close()
return self._job_results
def _job_archive_name(self):
return self._job_bundler.job_archive_name()
def upload_to_result_bucket(self):
self._bucket_upload_from_file(self._result_bucket)
def _run(self):
self._bucket_upload_from_file(self._code_bucket)
def _bucket_upload_from_file(self, bucket):
with open(self._job_archive(), 'rb') as file:
bucket.upload_from_file(self._job_archive_name(), file)
def _job_archive_name(self):
return self._job_bundler.job_archive_name()
def _job_archive(self):
return self._job_bundler.job_archive()
def test_unbundle_opens_correct_file(self):
job_bundler = JobBundler('fake_name', {}, None, None)
job_bundler.unbundle()
self.mock_tarfile_open.assert_called_with(self.temp_directory + '/fake_name.tgz', 'r:gz')
def test_job_archive_method_returns_job_archive(self):
job_bundler = JobBundler('fake_name', {}, None, None)
self.assertEqual(job_bundler.job_archive(), self.temp_directory + '/fake_name.tgz')
def test_job_name_method_returns_job_name(self):
job_bundler = JobBundler('fake_name', {}, None, None)
self.assertEqual(job_bundler.job_name(), 'fake_name')
def test_bundle_job_opens_file(self):
mock_job_source_bundle = self.MockJobSourceBundle('fake_source_archive_name')
job_bundler = JobBundler('fake_name', self._default_config, None, mock_job_source_bundle)
job_bundler._bundle_job()
self.mock_tarfile_open.assert_called_with(self.temp_directory + '/fake_name.tgz', 'w:gz')
def test_save_config_opens_file(self):
mock_job = self._create_mock_job()
job_bundler = JobBundler('fake_name', {}, mock_job, None)
job_bundler._save_config()
self.mock_builtins_open.assert_called_with(self.temp_directory + '/fake_name.config.yaml', 'w+')
def test_save_job_opens_file(self):
mock_job = self._create_mock_job()
job_bundler = JobBundler('fake_name', {}, mock_job, None)
job_bundler._save_job()
self.mock_builtins_open.assert_called_with('fake_name.bin', 'w+b')
def test_unbundle_extracts_from_tarfile(self):
return_object = self.MockContextManager()
self.mock_tarfile_open.return_value = return_object
job_bundler = JobBundler('fake_name', {}, None, None)
job_bundler.unbundle()
return_object.extractall.assert_called()